You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/09/02 01:35:09 UTC
(iotdb) branch rc/1.3.5 updated (25ecfcc1c7f -> fedfa10fc8d)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 25ecfcc1c7f [To dev/1.3] Pipe: Add a tool for validating and repairing isGeneratedByPipe mark in tsfile resources (#15934)
new bf809bae068 [To dev/1.3] Fixed multiple bugs of pipe
new 6405a06885f [TIMECHODB] Continue to revert the local persisting logic
new 2e339af15f6 Pipe: Added rate limiter for tsFile sending (#15765) (#15947)
new 097f725a901 Pipe: Fixed the ping-pong of certain realtime tsFiles (#15949)
new 1fbbdb3bece [To dev/1.3] Pipe: Fix DataNodeShutdownHook waiting report logic and add capture history file log (#15952) (#15951)
new 1ee10537dda Revert "Pipe: Added rate limiter for tsFile sending (#15765) (#15947)"
new ebdf275a350 [To dev/1.3] Pipe: Added progressIndex reset function to the resource resetting script (#15957)
new 7e403f4f5d6 [to dev/1.3] Set global time filter before analyzing last query
new 968d65cba04 [To dev/1.3] Optimize partition cache getRegionReplicaSet interface performance by batching
new 9da931acd60 [To dev/1.3] Pipe: Fixed the issue that frequent logins on the receiving end caused indicator leaks #16076 (#16085)
new fedfa10fc8d [To dev/1.3] Pipe: Reduced the progress index report interval & Added some logs (#15905) (#15908)
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 2 +
.../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 6 +-
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeClusterIT.java | 6 +-
.../IoTDBPipeConnectorCompressionIT.java | 7 +-
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 1 -
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeWithLoadIT.java | 6 +-
.../pipe/it/manual/AbstractPipeDualManualIT.java | 6 +-
.../manual/IoTDBPipeTypeConversionISessionIT.java | 12 +-
.../pipe/it/manual/IoTDBPipeTypeConversionIT.java | 2 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 1 +
.../it/cluster/IoTDBSubscriptionRestartIT.java | 3 +-
.../it/dual/AbstractSubscriptionDualIT.java | 3 +
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 3 +
.../it/dual/IoTDBSubscriptionTimePrecisionIT.java | 2 +
.../it/dual/IoTDBSubscriptionTopicIT.java | 3 +
.../it/local/AbstractSubscriptionLocalIT.java | 7 +-
.../it/triple/AbstractSubscriptionTripleIT.java | 4 +
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 6 +-
.../event/dml/insertion/TsFileInsertionEvent.java | 10 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../agent/task/PipeConfigNodeSubtaskExecutor.java | 4 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 28 +-
.../runtime/heartbeat/PipeHeartbeatParser.java | 2 -
.../confignode/persistence/pipe/PipeTaskInfo.java | 7 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 10 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 14 +-
.../confignode/service/ConfigNodeShutdownHook.java | 3 -
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +-
.../consensus/response/pipe/PipeTableRespTest.java | 6 +-
.../agent/PipeConfigNodeSubtaskExecutorTest.java | 3 +-
.../connector/PipeConfigNodeThriftRequestTest.java | 4 -
.../iotdb/confignode/persistence/PipeInfoTest.java | 4 +-
...pipe.bat => reset-resource-pipe-statistics.bat} | 4 +-
...y-pipe.sh => reset-resource-pipe-statistics.sh} | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 30 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 -
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 585 ++++++++++----------
.../task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../execution/PipeConnectorSubtaskExecutor.java | 8 +-
.../execution/PipeProcessorSubtaskExecutor.java | 2 +-
.../task/execution/PipeSubtaskExecutorManager.java | 18 +-
.../agent/task/stage/PipeTaskConnectorStage.java | 6 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 5 +
.../connector/PipeConnectorSubtaskManager.java | 19 +-
.../PipeRealtimePriorityBlockingQueue.java | 151 +++++-
.../client/IoTDBDataNodeAsyncClientManager.java | 36 ++
.../evolvable/batch/PipeTabletEventBatch.java | 39 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 10 +-
.../batch/PipeTabletEventTsFileBatch.java | 7 +
.../evolvable/request/PipeTransferPlanNodeReq.java | 1 -
.../request/PipeTransferTabletBatchReq.java | 1 -
.../request/PipeTransferTabletBinaryReq.java | 1 -
.../request/PipeTransferTabletInsertNodeReq.java | 1 -
.../request/PipeTransferTabletRawReq.java | 1 -
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +-
.../PipeConsensusTransferBatchReqBuilder.java | 3 +-
.../async/IoTDBDataRegionAsyncConnector.java | 91 ++--
.../PipeTransferTabletInsertionEventHandler.java | 2 +-
.../async/handler/PipeTransferTsFileHandler.java | 45 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 3 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../consensus/PipeConsensusSubtaskExecutor.java | 6 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 124 ++---
.../event/common/terminate/PipeTerminateEvent.java | 44 --
.../tsfile/PipeCompactedTsFileInsertionEvent.java | 252 +++++++++
.../common/tsfile/PipeTsFileInsertionEvent.java | 129 ++---
.../TsFileInsertionDataContainerProvider.java | 7 +-
.../event/realtime/PipeRealtimeEventFactory.java | 16 +-
.../dataregion/DataRegionWatermarkInjector.java | 4 -
.../dataregion/IoTDBDataRegionExtractor.java | 107 ++--
.../PipeHistoricalDataRegionTsFileExtractor.java | 162 ++----
.../realtime/PipeRealtimeDataRegionExtractor.java | 35 ++
.../PipeRealtimeDataRegionHybridExtractor.java | 180 ++-----
.../PipeRealtimeDataRegionLogExtractor.java | 5 +-
.../PipeRealtimeDataRegionTsFileExtractor.java | 7 +
.../realtime/assigner/PipeDataRegionAssigner.java | 50 +-
.../PipeTsFileEpochProgressIndexKeeper.java | 30 +-
.../dataregion/realtime/epoch/TsFileEpoch.java | 17 +-
.../realtime/epoch/TsFileEpochManager.java | 4 +-
.../listener/PipeInsertionDataNodeListener.java | 14 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 3 -
.../PipeDataNodeRemainingEventAndTimeOperator.java | 22 +-
.../overview/PipeDataNodeSinglePipeMetrics.java | 84 ++-
.../pipe/metric/overview/PipeResourceMetrics.java | 39 +-
.../overview/PipeWALInsertNodeCacheMetrics.java | 86 ---
.../sink/PipeDataRegionConnectorMetrics.java | 34 ++
.../twostage/exchange/payload/CombineRequest.java | 1 -
.../payload/FetchCombineResultRequest.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 40 +-
.../pipe/resource/PipeDataNodeResourceManager.java | 25 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 33 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 12 +-
.../resource/tsfile/PipeCompactionManager.java | 68 +++
...Resource.java => PipeTsFilePublicResource.java} | 158 +-----
.../pipe/resource/tsfile/PipeTsFileResource.java | 253 +--------
.../resource/tsfile/PipeTsFileResourceManager.java | 329 ++++++------
.../db/pipe/resource/wal/PipeWALResource.java | 169 ------
.../pipe/resource/wal/PipeWALResourceManager.java | 142 -----
.../wal/hardlink/PipeWALHardlinkResource.java | 47 --
.../hardlink/PipeWALHardlinkResourceManager.java | 152 ------
.../wal/selfhost/PipeWALSelfHostResource.java | 41 --
.../selfhost/PipeWALSelfHostResourceManager.java | 38 --
.../impl/DataNodeInternalRPCServiceImpl.java | 95 +++-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 1 +
.../plan/analyze/ClusterPartitionFetcher.java | 54 +-
.../plan/analyze/LoadTsFileAnalyzer.java | 34 +-
.../analyze/cache/partition/PartitionCache.java | 347 ++++++------
.../analyze/lock/DataNodeSchemaLockManager.java | 1 +
.../scheduler/load/LoadTsFileDispatcherImpl.java | 1 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 2 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 5 +-
.../iotdb/db/storageengine/StorageEngine.java | 8 -
.../db/storageengine/dataregion/DataRegion.java | 5 +-
.../task/InsertionCrossSpaceCompactionTask.java | 4 +-
.../compaction/execute/utils/CompactionUtils.java | 7 +-
.../dataregion/flush/NotifyFlushMemTable.java | 7 -
.../dataregion/memtable/AbstractMemTable.java | 13 -
.../dataregion/memtable/IMemTable.java | 4 -
.../dataregion/memtable/TsFileProcessor.java | 38 +-
.../dataregion/tsfile/TsFileManager.java | 13 +
.../dataregion/tsfile/TsFileResource.java | 39 +-
.../dataregion/wal/buffer/WALBuffer.java | 6 -
.../dataregion/wal/buffer/WALEntry.java | 4 +-
.../wal/checkpoint/CheckpointManager.java | 70 +--
.../dataregion/wal/checkpoint/MemTableInfo.java | 19 +-
.../dataregion/wal/io/WALByteBufReader.java | 13 -
.../dataregion/wal/node/WALFakeNode.java | 4 +-
.../storageengine/dataregion/wal/node/WALNode.java | 96 +---
.../dataregion/wal/utils/WALEntryHandler.java | 233 --------
.../dataregion/wal/utils/WALEntryPosition.java | 229 --------
.../dataregion/wal/utils/WALInsertNodeCache.java | 335 ------------
.../wal/utils/listener/WALFlushListener.java | 12 +-
.../db/storageengine/load/LoadTsFileManager.java | 1 +
...eeStatementDataTypeConvertExecutionVisitor.java | 169 ++----
.../converter/LoadTsFileDataTypeConverter.java | 54 +-
.../execution/SubscriptionSubtaskExecutor.java | 9 +-
.../task/stage/SubscriptionTaskConnectorStage.java | 4 +-
...va => TsFileResourcePipeStatisticsSetTool.java} | 125 +++--
.../connector/PipeDataNodeThriftRequestTest.java | 12 -
.../event/TsFileInsertionDataContainerTest.java | 16 +-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 101 ++--
.../resource/PipeTsFileResourceManagerTest.java | 96 ++--
.../PipeWALHardlinkResourceManagerTest.java | 114 ----
.../plan/analyze/cache/PartitionCacheTest.java | 5 +-
.../TsFileResourceProgressIndexTest.java | 23 +-
.../wal/checkpoint/CheckpointManagerTest.java | 172 ------
.../dataregion/wal/node/WALEntryHandlerTest.java | 586 ---------------------
.../dataregion/wal/node/WALNodeTest.java | 17 -
.../wal/node/WalDeleteOutdatedNewTest.java | 65 ---
.../wal/utils/WALInsertNodeCacheTest.java | 190 -------
.../commons/concurrent/IoTDBThreadPoolFactory.java | 15 +
.../iotdb/commons/concurrent/ThreadName.java | 10 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 407 +++++---------
.../commons/consensus/index/ProgressIndex.java | 2 +-
.../consensus/index/impl/IoTProgressIndex.java | 27 +-
.../agent/runtime/PipePeriodicalJobExecutor.java | 33 --
.../commons/pipe/agent/task/PipeTaskAgent.java | 61 ++-
.../agent/task/execution/PipeSubtaskExecutor.java | 45 +-
.../commons/pipe/agent/task/meta/PipeMeta.java | 6 +-
.../pipe/agent/task/meta/PipeRuntimeMeta.java | 36 +-
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 141 +----
.../task/progress/PipeEventCommitManager.java | 25 +-
.../agent/task/progress/PipeEventCommitter.java | 12 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 5 -
.../agent/task/subtask/PipeReportableSubtask.java | 24 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 160 ++----
.../iotdb/commons/pipe/config/PipeDescriptor.java | 129 ++---
.../config/constant/PipeConnectorConstant.java | 2 +-
.../thrift/request/PipeTransferFilePieceReq.java | 1 -
.../thrift/request/PipeTransferFileSealReqV1.java | 1 -
.../thrift/request/PipeTransferFileSealReqV2.java | 1 -
.../thrift/request/PipeTransferHandshakeV1Req.java | 1 -
.../thrift/request/PipeTransferHandshakeV2Req.java | 1 -
.../thrift/request/PipeTransferSliceReq.java | 1 -
.../pipe/connector/protocol/IoTDBConnector.java | 12 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 13 +
.../commons/pipe/resource/log/PipeLogManager.java | 2 +-
.../commons/pipe/resource/log/PipeLogStatus.java | 6 +-
.../iotdb/commons/service/metric/enums/Metric.java | 9 +-
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 22 +-
194 files changed, 2994 insertions(+), 6033 deletions(-)
rename iotdb-core/datanode/src/assembly/resources/tools/tsfile/{mark-is-generated-by-pipe.bat => reset-resource-pipe-statistics.bat} (92%)
rename iotdb-core/datanode/src/assembly/resources/tools/tsfile/{mark-is-generated-by-pipe.sh => reset-resource-pipe-statistics.sh} (90%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/{PipeTsFileResource.java => PipeTsFilePublicResource.java} (63%)
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/{TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool.java => TsFileResourcePipeStatisticsSetTool.java} (60%)
delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeWALHardlinkResourceManagerTest.java
delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManagerTest.java
delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALEntryHandlerTest.java
delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCacheTest.java
(iotdb) 02/11: [TIMECHODB] Continue to revert the local persisting logic
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6405a06885fd5c420650b2b89c4c7773e29d790f
Author: 陈 哲涵 <13...@qq.com>
AuthorDate: Tue Jul 15 10:55:27 2025 +0000
[TIMECHODB] Continue to revert the local persisting logic
---
.../runtime/heartbeat/PipeHeartbeatParser.java | 2 -
.../confignode/persistence/pipe/PipeTaskInfo.java | 7 +-
.../impl/pipe/task/AlterPipeProcedureV2.java | 10 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 14 +-
.../confignode/service/ConfigNodeShutdownHook.java | 3 -
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +-
.../consensus/response/pipe/PipeTableRespTest.java | 6 +-
.../agent/PipeConfigNodeSubtaskExecutorTest.java | 3 +-
.../iotdb/confignode/persistence/PipeInfoTest.java | 4 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 5 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 3 -
.../agent/runtime/PipePeriodicalJobExecutor.java | 33 -----
.../commons/pipe/agent/task/PipeTaskAgent.java | 3 -
.../commons/pipe/agent/task/meta/PipeMeta.java | 6 +-
.../pipe/agent/task/meta/PipeRuntimeMeta.java | 36 ++----
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 141 +--------------------
.../iotdb/commons/pipe/task/PipeMetaDeSerTest.java | 22 +---
17 files changed, 42 insertions(+), 268 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
index 6f6c0b2d443..e2303fecdea 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java
@@ -244,7 +244,6 @@ public class PipeHeartbeatParser {
.equals(PipeStatus.STOPPED)) {
PipeRuntimeMeta runtimeMeta = pipeMetaFromCoordinator.getRuntimeMeta();
runtimeMeta.getStatus().set(PipeStatus.STOPPED);
- runtimeMeta.onSetPipeDroppedOrStopped();
runtimeMeta.setIsStoppedByRuntimeException(true);
needWriteConsensusOnConfigNodes.set(true);
@@ -274,7 +273,6 @@ public class PipeHeartbeatParser {
exceptionMap.put(nodeId, exception);
}
runtimeMeta.getStatus().set(PipeStatus.STOPPED);
- runtimeMeta.onSetPipeDroppedOrStopped();
runtimeMeta.setIsStoppedByRuntimeException(true);
needWriteConsensusOnConfigNodes.set(true);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
index 880271f8d01..372209bfd74 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java
@@ -616,11 +616,7 @@ public class PipeTaskInfo implements SnapshotProcessor {
if (newLeader != -1) {
consensusGroupIdToTaskMetaMap.put(
consensusGroupId.getId(),
- new PipeTaskMeta(
- MinimumProgressIndex.INSTANCE,
- newLeader,
- consensusGroupId.getId(),
- false));
+ new PipeTaskMeta(MinimumProgressIndex.INSTANCE, newLeader));
}
// else:
// "The pipe task meta does not contain the data region group {} or
@@ -794,7 +790,6 @@ public class PipeTaskInfo implements SnapshotProcessor {
// Mark the status of the pipe with exception as stopped
runtimeMeta.getStatus().set(PipeStatus.STOPPED);
- runtimeMeta.onSetPipeDroppedOrStopped();
runtimeMeta.setIsStoppedByRuntimeException(true);
final Map<Integer, PipeRuntimeException> exceptionMap =
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java
index b17795afa43..b11c74408a6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java
@@ -155,11 +155,7 @@ public class AlterPipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
// Pipe only collect user's data, filter metric database here.
updatedConsensusGroupIdToTaskMetaMap.put(
regionGroupId.getId(),
- new PipeTaskMeta(
- currentPipeTaskMeta.getProgressIndex(),
- regionLeaderNodeId,
- regionGroupId.getId(),
- false));
+ new PipeTaskMeta(currentPipeTaskMeta.getProgressIndex(), regionLeaderNodeId));
}
});
@@ -174,9 +170,7 @@ public class AlterPipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
new PipeTaskMeta(
configRegionTaskMeta.getProgressIndex(),
// The leader of the config region is the config node itself
- ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(),
- Integer.MIN_VALUE,
- false));
+ ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId()));
}
updatedPipeRuntimeMeta = new PipeRuntimeMeta(updatedConsensusGroupIdToTaskMetaMap);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java
index 3b91aa07802..813d4ebe69e 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java
@@ -171,9 +171,7 @@ public class CreatePipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
groupId.getId(),
new PipeTaskMeta(
new RecoverProgressIndex(senderDataNodeId, new SimpleProgressIndex(0, 0)),
- senderDataNodeId,
- groupId.getId(),
- false));
+ senderDataNodeId));
} else {
// data regions & schema regions
env.getConfigManager()
@@ -189,11 +187,7 @@ public class CreatePipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
// Pipe only collect user's data, filter out metric database here.
consensusGroupIdToTaskMetaMap.put(
regionGroupId.getId(),
- new PipeTaskMeta(
- MinimumProgressIndex.INSTANCE,
- regionLeaderNodeId,
- regionGroupId.getId(),
- false));
+ new PipeTaskMeta(MinimumProgressIndex.INSTANCE, regionLeaderNodeId));
}
});
@@ -206,9 +200,7 @@ public class CreatePipeProcedureV2 extends AbstractOperatePipeProcedureV2 {
new PipeTaskMeta(
MinimumProgressIndex.INSTANCE,
// The leader of the config region is the config node itself
- ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId(),
- Integer.MIN_VALUE,
- false));
+ ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId()));
}
pipeRuntimeMeta = new PipeRuntimeMeta(consensusGroupIdToTaskMetaMap);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
index bd12adbd804..5c3ec5af063 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.pipe.agent.runtime.PipePeriodicalJobExecutor;
import org.apache.iotdb.confignode.client.CnToCnNodeRequestType;
import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool;
import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
@@ -88,8 +87,6 @@ public class ConfigNodeShutdownHook extends Thread {
"Reporting ConfigNode shutdown failed. The cluster will still take the current ConfigNode as Running for a few seconds.");
}
}
- // Shutdown pipe progressIndex background service
- PipePeriodicalJobExecutor.shutdownBackgroundService();
if (LOGGER.isInfoEnabled()) {
LOGGER.info(
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java
index c09dafb1d29..de2a350571d 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java
@@ -849,7 +849,7 @@ public class ConfigPhysicalPlanSerDeTest {
extractorAttributes.put("extractor", "org.apache.iotdb.pipe.extractor.DefaultExtractor");
processorAttributes.put("processor", "org.apache.iotdb.pipe.processor.SDTFilterProcessor");
connectorAttributes.put("connector", "org.apache.iotdb.pipe.protocol.ThriftTransporter");
- final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
ConcurrentMap<Integer, PipeTaskMeta> pipeTasks = new ConcurrentHashMap<>();
pipeTasks.put(1, pipeTaskMeta);
PipeStaticMeta pipeStaticMeta =
@@ -874,7 +874,7 @@ public class ConfigPhysicalPlanSerDeTest {
extractorAttributes.put("pattern", "root.db");
processorAttributes.put("processor", "do-nothing-processor");
connectorAttributes.put("batch.enable", "false");
- final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
final ConcurrentMap<Integer, PipeTaskMeta> pipeTasks = new ConcurrentHashMap<>();
pipeTasks.put(1, pipeTaskMeta);
PipeStaticMeta pipeStaticMeta =
@@ -912,7 +912,7 @@ public class ConfigPhysicalPlanSerDeTest {
@Test
public void OperateMultiplePipesPlanV2Test() throws IOException {
- final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
final ConcurrentMap<Integer, PipeTaskMeta> pipeTasks = new ConcurrentHashMap<>();
pipeTasks.put(1, pipeTaskMeta);
PipeStaticMeta pipeStaticMeta =
@@ -925,7 +925,7 @@ public class ConfigPhysicalPlanSerDeTest {
PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta(pipeTasks);
CreatePipePlanV2 createPipePlanV2 = new CreatePipePlanV2(pipeStaticMeta, pipeRuntimeMeta);
- final PipeTaskMeta pipeTaskMeta1 = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 2, 2, false);
+ final PipeTaskMeta pipeTaskMeta1 = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 2);
final ConcurrentMap<Integer, PipeTaskMeta> pipeTasks1 = new ConcurrentHashMap<>();
pipeTasks.put(2, pipeTaskMeta1);
PipeStaticMeta pipeStaticMeta1 =
@@ -1024,8 +1024,8 @@ public class ConfigPhysicalPlanSerDeTest {
new PipeRuntimeMeta(
new ConcurrentHashMap<Integer, PipeTaskMeta>() {
{
- put(456, new PipeTaskMeta(new IoTProgressIndex(1, 2L), 987, 1, false));
- put(123, new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 789, 1, false));
+ put(456, new PipeTaskMeta(new IoTProgressIndex(1, 2L), 987));
+ put(123, new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 789));
}
});
pipeMetaList.add(new PipeMeta(pipeStaticMeta, pipeRuntimeMeta));
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java
index 04dea675015..94189a19d99 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java
@@ -54,7 +54,7 @@ public class PipeTableRespTest {
connectorAttributes.put("host", "127.0.0.1");
connectorAttributes.put("port", "6667");
- PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
ConcurrentMap<Integer, PipeTaskMeta> pipeTasks = new ConcurrentHashMap<>();
pipeTasks.put(1, pipeTaskMeta);
PipeStaticMeta pipeStaticMeta =
@@ -74,7 +74,7 @@ public class PipeTableRespTest {
connectorAttributes1.put("host", "127.0.0.1");
connectorAttributes1.put("port", "6667");
- PipeTaskMeta pipeTaskMeta1 = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ PipeTaskMeta pipeTaskMeta1 = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
ConcurrentMap<Integer, PipeTaskMeta> pipeTasks1 = new ConcurrentHashMap<>();
pipeTasks1.put(1, pipeTaskMeta1);
PipeStaticMeta pipeStaticMeta1 =
@@ -94,7 +94,7 @@ public class PipeTableRespTest {
connectorAttributes2.put("host", "172.30.30.30");
connectorAttributes2.put("port", "6667");
- PipeTaskMeta pipeTaskMeta2 = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ PipeTaskMeta pipeTaskMeta2 = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
ConcurrentMap<Integer, PipeTaskMeta> pipeTasks2 = new ConcurrentHashMap<>();
pipeTasks2.put(1, pipeTaskMeta2);
PipeStaticMeta pipeStaticMeta2 =
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeSubtaskExecutorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeSubtaskExecutorTest.java
index a43a87b120e..f2fa5b0205a 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeSubtaskExecutorTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/PipeConfigNodeSubtaskExecutorTest.java
@@ -62,8 +62,7 @@ public class PipeConfigNodeSubtaskExecutorTest {
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName());
}
},
- new PipeTaskMeta(
- MinimumProgressIndex.INSTANCE, Integer.MIN_VALUE, Integer.MIN_VALUE, false)));
+ new PipeTaskMeta(MinimumProgressIndex.INSTANCE, Integer.MIN_VALUE)));
}
@After
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PipeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PipeInfoTest.java
index 815d5c1757e..c3e7916108f 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PipeInfoTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PipeInfoTest.java
@@ -86,7 +86,7 @@ public class PipeInfoTest {
connectorAttributes.put("host", "127.0.0.1");
connectorAttributes.put("port", "6667");
- PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
ConcurrentMap<Integer, PipeTaskMeta> pipeTasks = new ConcurrentHashMap<>();
pipeTasks.put(1, pipeTaskMeta);
PipeStaticMeta pipeStaticMeta =
@@ -121,7 +121,7 @@ public class PipeInfoTest {
extractorAttributes.put("extractor", "org.apache.iotdb.pipe.extractor.DefaultExtractor");
processorAttributes.put("processor", "org.apache.iotdb.pipe.processor.SDTFilterProcessor");
connectorAttributes.put("connector", "org.apache.iotdb.pipe.protocol.ThriftTransporter");
- PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1, 1, false);
+ PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
ConcurrentMap<Integer, PipeTaskMeta> pipeTasks = new ConcurrentHashMap<>();
pipeTasks.put(1, pipeTaskMeta);
PipeStaticMeta pipeStaticMeta =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
index 41cc715eb52..c1c4893ae8a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -261,10 +261,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
pipeName = environment.getPipeName();
creationTime = environment.getCreationTime();
pipeTaskMeta = environment.getPipeTaskMeta();
-
- // progressIndex is immutable in `updateToMinimumEqualOrIsAfterProgressIndex`, so data
- // consistency in `environment.getPipeTaskMeta().getProgressIndex()` is ensured.
- startIndex = environment.getPipeTaskMeta().restoreProgressIndex();
+ startIndex = environment.getPipeTaskMeta().getProgressIndex();
dataRegionId = environment.getRegionId();
pipePattern = PipePattern.parsePipePatternFromSourceParameters(parameters);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
index 8dd3d39c92c..5b62a0d614c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.pipe.agent.runtime.PipePeriodicalJobExecutor;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.consensus.exception.ConsensusException;
@@ -123,8 +122,6 @@ public class DataNodeShutdownHook extends Thread {
}
// Persist progress index before shutdown to accurate recovery after restart
PipeDataNodeAgent.task().persistAllProgressIndex();
- // Shutdown pipe progressIndex background service
- PipePeriodicalJobExecutor.shutdownBackgroundService();
// Actually stop all services started by the DataNode.
// If we don't call this, services like the RestService are not stopped and I can't re-start
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/runtime/PipePeriodicalJobExecutor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/runtime/PipePeriodicalJobExecutor.java
index 33ac03c5c96..3226b3947f0 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/runtime/PipePeriodicalJobExecutor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/runtime/PipePeriodicalJobExecutor.java
@@ -21,16 +21,8 @@ package org.apache.iotdb.commons.pipe.agent.runtime;
import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.commons.concurrent.ThreadName;
-import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
/**
* The shortest scheduling cycle for these jobs is {@link
* PipeConfig#getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds()}, suitable for jobs that are
@@ -38,31 +30,6 @@ import java.util.concurrent.TimeUnit;
*/
public class PipePeriodicalJobExecutor extends AbstractPipePeriodicalJobExecutor {
- private static final Logger LOGGER = LoggerFactory.getLogger(PipePeriodicalJobExecutor.class);
- // This background service is used to execute jobs that need to be cancelled and released.
- private static final ScheduledExecutorService backgroundService =
- IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor(
- ThreadName.PIPE_PROGRESS_INDEX_BACKGROUND_SERVICE.getName());
-
- public static Future<?> submitBackgroundJob(
- Runnable job, long initialDelayInMs, long periodInMs) {
- return ScheduledExecutorUtil.safelyScheduleWithFixedDelay(
- backgroundService, job, initialDelayInMs, periodInMs, TimeUnit.MILLISECONDS);
- }
-
- public static void shutdownBackgroundService() {
- backgroundService.shutdownNow();
- try {
- if (!backgroundService.awaitTermination(30, TimeUnit.SECONDS)) {
- LOGGER.warn("Pipe progressIndex background service did not terminate within {}s", 30);
- }
- } catch (InterruptedException e) {
- LOGGER.warn(
- "Pipe progressIndex background service is interrupted while waiting for termination");
- Thread.currentThread().interrupt();
- }
- }
-
public PipePeriodicalJobExecutor() {
super(
IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
index 55aa6c5f17a..d1ea8807252 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
@@ -544,7 +544,6 @@ public abstract class PipeTaskAgent {
// but the pipe task meta has not been cleaned up (in case of failure when executing
// dropPipeTaskByConsensusGroup).
existedPipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED);
- existedPipeMeta.getRuntimeMeta().onSetPipeDroppedOrStopped();
// Drop pipe tasks
final Map<Integer, PipeTask> pipeTasks =
@@ -586,7 +585,6 @@ public abstract class PipeTaskAgent {
// but the pipe task meta has not been cleaned up (in case of failure when executing
// dropPipeTaskByConsensusGroup).
existedPipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED);
- existedPipeMeta.getRuntimeMeta().onSetPipeDroppedOrStopped();
// Drop pipe tasks
final Map<Integer, PipeTask> pipeTasks =
@@ -679,7 +677,6 @@ public abstract class PipeTaskAgent {
// Set pipe meta status to STOPPED
existedPipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.STOPPED);
- existedPipeMeta.getRuntimeMeta().onSetPipeDroppedOrStopped();
}
////////////////////////// Checker //////////////////////////
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java
index c71156a234b..997278010e9 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.commons.pipe.agent.task.meta;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-
import org.apache.tsfile.utils.PublicBAOS;
import java.io.DataOutputStream;
@@ -83,9 +81,7 @@ public class PipeMeta {
public static PipeMeta deserialize4TaskAgent(final ByteBuffer byteBuffer) {
final PipeStaticMeta staticMeta = PipeStaticMeta.deserialize(byteBuffer);
- final PipeRuntimeMeta runtimeMeta =
- PipeRuntimeMeta.deserialize(
- byteBuffer, PipeConfig.getInstance().isPipeProgressIndexPersistEnabled());
+ final PipeRuntimeMeta runtimeMeta = PipeRuntimeMeta.deserialize(byteBuffer);
return new PipeMeta(
staticMeta,
runtimeMeta,
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java
index 752edae0cf0..8c22a7bfd55 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeRuntimeMeta.java
@@ -110,10 +110,6 @@ public class PipeRuntimeMeta {
return status;
}
- public void onSetPipeDroppedOrStopped() {
- consensusGroupId2TaskMetaMap.values().forEach(PipeTaskMeta::cancelPersistProgressIndexFuture);
- }
-
public ConcurrentMap<Integer, PipeTaskMeta> getConsensusGroupId2TaskMetaMap() {
return consensusGroupId2TaskMetaMap;
}
@@ -200,9 +196,7 @@ public class PipeRuntimeMeta {
for (int i = 0; i < size; ++i) {
final int taskIndex = ReadWriteIOUtils.readInt(inputStream);
pipeRuntimeMeta.consensusGroupId2TaskMetaMap.put(
- taskIndex,
- PipeTaskMeta.deserialize(
- PipeRuntimeMetaVersion.VERSION_1, inputStream, taskIndex, false));
+ taskIndex, PipeTaskMeta.deserialize(PipeRuntimeMetaVersion.VERSION_1, inputStream));
}
return pipeRuntimeMeta;
@@ -217,9 +211,7 @@ public class PipeRuntimeMeta {
for (int i = 0; i < size; ++i) {
final int taskIndex = ReadWriteIOUtils.readInt(inputStream);
pipeRuntimeMeta.consensusGroupId2TaskMetaMap.put(
- taskIndex,
- PipeTaskMeta.deserialize(
- PipeRuntimeMetaVersion.VERSION_2, inputStream, taskIndex, false));
+ taskIndex, PipeTaskMeta.deserialize(PipeRuntimeMetaVersion.VERSION_2, inputStream));
}
size = ReadWriteIOUtils.readInt(inputStream);
@@ -235,20 +227,15 @@ public class PipeRuntimeMeta {
return pipeRuntimeMeta;
}
- public static PipeRuntimeMeta deserialize(ByteBuffer byteBuffer) {
- return deserialize(byteBuffer, false);
- }
-
- public static PipeRuntimeMeta deserialize(
- final ByteBuffer byteBuffer, final boolean needPersist) {
+ public static PipeRuntimeMeta deserialize(final ByteBuffer byteBuffer) {
final byte pipeRuntimeVersionByte = ReadWriteIOUtils.readByte(byteBuffer);
final PipeRuntimeMetaVersion pipeRuntimeMetaVersion =
PipeRuntimeMetaVersion.deserialize(pipeRuntimeVersionByte);
switch (pipeRuntimeMetaVersion) {
case VERSION_1:
- return deserializeVersion1(byteBuffer, pipeRuntimeVersionByte, needPersist);
+ return deserializeVersion1(byteBuffer, pipeRuntimeVersionByte);
case VERSION_2:
- return deserializeVersion2(byteBuffer, needPersist);
+ return deserializeVersion2(byteBuffer);
default:
throw new UnsupportedOperationException(
"Unknown pipe runtime meta version: " + pipeRuntimeMetaVersion.getVersion());
@@ -256,7 +243,7 @@ public class PipeRuntimeMeta {
}
private static PipeRuntimeMeta deserializeVersion1(
- ByteBuffer byteBuffer, byte pipeRuntimeVersionByte, final boolean needPersist) {
+ ByteBuffer byteBuffer, byte pipeRuntimeVersionByte) {
final PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta();
pipeRuntimeMeta.status.set(PipeStatus.getPipeStatus(pipeRuntimeVersionByte));
@@ -265,16 +252,13 @@ public class PipeRuntimeMeta {
for (int i = 0; i < size; ++i) {
final int taskIndex = ReadWriteIOUtils.readInt(byteBuffer);
pipeRuntimeMeta.consensusGroupId2TaskMetaMap.put(
- taskIndex,
- PipeTaskMeta.deserialize(
- PipeRuntimeMetaVersion.VERSION_1, byteBuffer, taskIndex, needPersist));
+ taskIndex, PipeTaskMeta.deserialize(PipeRuntimeMetaVersion.VERSION_1, byteBuffer));
}
return pipeRuntimeMeta;
}
- public static PipeRuntimeMeta deserializeVersion2(
- ByteBuffer byteBuffer, final boolean needPersist) {
+ public static PipeRuntimeMeta deserializeVersion2(ByteBuffer byteBuffer) {
final PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta();
pipeRuntimeMeta.status.set(PipeStatus.getPipeStatus(ReadWriteIOUtils.readByte(byteBuffer)));
@@ -283,9 +267,7 @@ public class PipeRuntimeMeta {
for (int i = 0; i < size; ++i) {
final int taskIndex = ReadWriteIOUtils.readInt(byteBuffer);
pipeRuntimeMeta.consensusGroupId2TaskMetaMap.put(
- taskIndex,
- PipeTaskMeta.deserialize(
- PipeRuntimeMetaVersion.VERSION_2, byteBuffer, taskIndex, needPersist));
+ taskIndex, PipeTaskMeta.deserialize(PipeRuntimeMetaVersion.VERSION_2, byteBuffer));
}
size = ReadWriteIOUtils.readInt(byteBuffer);
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java
index 6a4ab25db7e..638d879bfa7 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeTaskMeta.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.commons.pipe.agent.task.meta;
-import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.consensus.index.ProgressIndexType;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeConnectorCriticalException;
@@ -27,51 +26,30 @@ import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeExceptionType;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
-import org.apache.iotdb.commons.pipe.agent.runtime.PipePeriodicalJobExecutor;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.commons.io.FileUtils;
-import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
public class PipeTaskMeta {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTaskMeta.class);
- private static final String PREFIX = "__progressIndex_";
private final AtomicReference<ProgressIndex> progressIndex = new AtomicReference<>();
private final AtomicInteger leaderNodeId = new AtomicInteger(0);
- private final AtomicLong updateCount = new AtomicLong(0);
- private final AtomicLong lastPersistCount = new AtomicLong(0);
- private final long checkPointGap =
- PipeConfig.getInstance().getPipeProgressIndexPersistCheckPointGap();
- private File progressIndexPersistFile;
- private final AtomicBoolean isRegisterPersistTask = new AtomicBoolean(false);
- private Future<?> persistProgressIndexFuture;
-
/**
* Stores the exceptions encountered during run time of each pipe task.
*
@@ -84,26 +62,9 @@ public class PipeTaskMeta {
private final Set<PipeRuntimeException> exceptionMessages =
Collections.newSetFromMap(new ConcurrentHashMap<>());
- public PipeTaskMeta(
- /* @NotNull */ final ProgressIndex progressIndex,
- final int leaderNodeId,
- final int taskIndex,
- final boolean needPersistProgressIndex) {
+ public PipeTaskMeta(/* @NotNull */ final ProgressIndex progressIndex, final int leaderNodeId) {
this.progressIndex.set(progressIndex);
this.leaderNodeId.set(leaderNodeId);
- // PipeTaskMeta created in configNode doesn't need to persist progress index.
- if (needPersistProgressIndex) {
- this.progressIndexPersistFile =
- new File(
- IoTDBConstant.DN_DEFAULT_DATA_DIR
- + File.separator
- + IoTDBConstant.SYSTEM_FOLDER_NAME
- + File.separator
- + PipeConfig.getInstance().getPipeHardlinkBaseDirName()
- + File.separator
- + PipeConfig.getInstance().getPipeProgressIndexPersistDirName(),
- PREFIX + taskIndex);
- }
}
public ProgressIndex getProgressIndex() {
@@ -111,89 +72,8 @@ public class PipeTaskMeta {
}
public ProgressIndex updateProgressIndex(final ProgressIndex updateIndex) {
- // only pipeTaskMeta that need to updateProgressIndex will persist progress index
- // isRegisterPersistTask is used to avoid multiple threads registering persist task concurrently
- if (Objects.nonNull(progressIndexPersistFile)
- && !isRegisterPersistTask.getAndSet(true)
- && this.persistProgressIndexFuture == null
- && PipeConfig.getInstance().isPipeProgressIndexPersistEnabled()) {
- this.persistProgressIndexFuture =
- PipePeriodicalJobExecutor.submitBackgroundJob(
- () -> {
- if (PipeConfig.getInstance().isPipeProgressIndexPersistEnabled()) {
- persistProgressIndex();
- }
- },
- 0,
- PipeConfig.getInstance().getPipeProgressIndexFlushIntervalMs());
- }
-
- progressIndex.updateAndGet(
+ return progressIndex.updateAndGet(
index -> index.updateToMinimumEqualOrIsAfterProgressIndex(updateIndex));
- if (Objects.nonNull(progressIndexPersistFile)
- && updateCount.incrementAndGet() - lastPersistCount.get() > checkPointGap
- && PipeConfig.getInstance().isPipeProgressIndexPersistEnabled()) {
- persistProgressIndex();
- }
- return progressIndex.get();
- }
-
- private synchronized void persistProgressIndex() {
- if (lastPersistCount.get() == updateCount.get()) {
- // in case of multiple threads calling updateProgressIndex at the same time
- return;
- }
-
- try (final PublicBAOS byteArrayOutputStream = new PublicBAOS();
- final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
- progressIndex.get().serialize(outputStream);
- // append is false by default.
- FileUtils.writeByteArrayToFile(
- progressIndexPersistFile,
- byteArrayOutputStream.getBuf(),
- 0,
- byteArrayOutputStream.size());
- lastPersistCount.set(updateCount.get());
- } catch (IOException e) {
- LOGGER.warn("Failed to persist progress index {} for {}", progressIndex.get(), this, e);
- }
- }
-
- public ProgressIndex restoreProgressIndex() {
- if (!progressIndexPersistFile.exists() || progressIndexPersistFile.length() == 0) {
- return progressIndex.get();
- }
-
- try {
- final byte[] fileData = Files.readAllBytes(progressIndexPersistFile.toPath());
-
- try (final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(fileData);
- final DataInputStream inputStream = new DataInputStream(byteArrayInputStream)) {
- final ProgressIndex restoredIndex = ProgressIndexType.deserializeFrom(inputStream);
- progressIndex.get().updateToMinimumEqualOrIsAfterProgressIndex(restoredIndex);
- LOGGER.info(
- "{} successfully restored progress index from [{}], current index: {}",
- this,
- progressIndexPersistFile.getAbsolutePath(),
- progressIndex.get());
- }
- } catch (final IOException e) {
- LOGGER.warn(
- "{} failed to restore progress index from [{}].",
- this,
- progressIndexPersistFile.getAbsolutePath(),
- e);
- }
- return progressIndex.get();
- }
-
- public void cancelPersistProgressIndexFuture() {
- if (Objects.nonNull(progressIndexPersistFile)
- && isRegisterPersistTask.getAndSet(false)
- && persistProgressIndexFuture != null) {
- persistProgressIndexFuture.cancel(false);
- persistProgressIndexFuture = null;
- }
}
public int getLeaderNodeId() {
@@ -245,16 +125,12 @@ public class PipeTaskMeta {
}
public static PipeTaskMeta deserialize(
- final PipeRuntimeMetaVersion version,
- final ByteBuffer byteBuffer,
- final int taskIndex,
- final boolean needPersist) {
+ final PipeRuntimeMetaVersion version, final ByteBuffer byteBuffer) {
final ProgressIndex progressIndex = ProgressIndexType.deserializeFrom(byteBuffer);
final int leaderNodeId = ReadWriteIOUtils.readInt(byteBuffer);
- final PipeTaskMeta pipeTaskMeta =
- new PipeTaskMeta(progressIndex, leaderNodeId, taskIndex, needPersist);
+ final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(progressIndex, leaderNodeId);
final int size = ReadWriteIOUtils.readInt(byteBuffer);
for (int i = 0; i < size; ++i) {
final PipeRuntimeException pipeRuntimeException =
@@ -265,17 +141,12 @@ public class PipeTaskMeta {
}
public static PipeTaskMeta deserialize(
- final PipeRuntimeMetaVersion version,
- final InputStream inputStream,
- final int taskIndex,
- final boolean needPersist)
- throws IOException {
+ final PipeRuntimeMetaVersion version, final InputStream inputStream) throws IOException {
final ProgressIndex progressIndex = ProgressIndexType.deserializeFrom(inputStream);
final int leaderNodeId = ReadWriteIOUtils.readInt(inputStream);
- final PipeTaskMeta pipeTaskMeta =
- new PipeTaskMeta(progressIndex, leaderNodeId, taskIndex, needPersist);
+ final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(progressIndex, leaderNodeId);
final int size = ReadWriteIOUtils.readInt(inputStream);
for (int i = 0; i < size; ++i) {
final PipeRuntimeException pipeRuntimeException =
diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java
index d4e6008547c..8e04baf7cc0 100644
--- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java
+++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaDeSerTest.java
@@ -97,27 +97,20 @@ public class PipeMetaDeSerTest {
new PipeRuntimeMeta(
new ConcurrentHashMap<Integer, PipeTaskMeta>() {
{
- put(123, new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 987, 123, false));
- put(234, new PipeTaskMeta(new IoTProgressIndex(1, 2L), 789, 234, false));
- put(345, new PipeTaskMeta(new SimpleProgressIndex(3, 4), 789, 345, false));
- put(456, new PipeTaskMeta(finalHybridProgressIndex, 789, 456, false));
+ put(123, new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 987));
+ put(234, new PipeTaskMeta(new IoTProgressIndex(1, 2L), 789));
+ put(345, new PipeTaskMeta(new SimpleProgressIndex(3, 4), 789));
+ put(456, new PipeTaskMeta(finalHybridProgressIndex, 789));
put(
567,
new PipeTaskMeta(
- new RecoverProgressIndex(1, new SimpleProgressIndex(1, 9)),
- 123,
- 567,
- false));
+ new RecoverProgressIndex(1, new SimpleProgressIndex(1, 9)), 123));
put(
678,
new PipeTaskMeta(
new TimeWindowStateProgressIndex(timeSeries2TimestampWindowBufferPairMap),
- 789,
- 678,
- false));
- put(
- Integer.MIN_VALUE,
- new PipeTaskMeta(new MetaProgressIndex(987), 0, Integer.MIN_VALUE, false));
+ 789));
+ put(Integer.MIN_VALUE, new PipeTaskMeta(new MetaProgressIndex(987), 0));
}
});
ByteBuffer runtimeByteBuffer = pipeRuntimeMeta.serialize();
@@ -136,7 +129,6 @@ public class PipeMetaDeSerTest {
Assert.assertEquals(pipeRuntimeMeta, pipeRuntimeMeta1);
pipeRuntimeMeta.getStatus().set(PipeStatus.DROPPED);
- pipeRuntimeMeta.onSetPipeDroppedOrStopped();
pipeRuntimeMeta.setIsStoppedByRuntimeException(true);
pipeRuntimeMeta.setExceptionsClearTime(0);
pipeRuntimeMeta
(iotdb) 05/11: [To dev/1.3] Pipe: Fix DataNodeShutdownHook waiting report logic and add capture history file log (#15952) (#15951)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1fbbdb3bece89b523ed0e23211289760a1455637
Author: Zhenyu Luo <lu...@gmail.com>
AuthorDate: Wed Jul 16 14:35:16 2025 +0800
[To dev/1.3] Pipe: Fix DataNodeShutdownHook waiting report logic and add capture history file log (#15952) (#15951)
* Pipe:Fix DataNodeShutdownHook waiting report logic and add capture history file log
* update log
---
.../PipeHistoricalDataRegionTsFileExtractor.java | 15 +++++++++++++--
.../org/apache/iotdb/db/service/DataNodeShutdownHook.java | 2 +-
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
index c1c4893ae8a..25759654a5b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -473,8 +473,19 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
if (startIndex instanceof StateProgressIndex) {
startIndex = ((StateProgressIndex) startIndex).getInnerProgressIndex();
}
- return !startIndex.isAfter(resource.getMaxProgressIndex())
- && !startIndex.equals(resource.getMaxProgressIndex());
+
+ if (!startIndex.isAfter(resource.getMaxProgressIndex())
+ && !startIndex.equals(resource.getMaxProgressIndex())) {
+ LOGGER.info(
+ "Pipe {}@{}: file {} meets mayTsFileContainUnprocessedData condition, extractor progressIndex: {}, resource ProgressIndex: {}",
+ pipeName,
+ dataRegionId,
+ resource.getTsFilePath(),
+ startIndex,
+ resource.getMaxProgressIndex());
+ return true;
+ }
+ return false;
}
private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
index 5b62a0d614c..1a6af1cf9b0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
@@ -97,7 +97,7 @@ public class DataNodeShutdownHook extends Thread {
PipeDataNodeSinglePipeMetrics.getInstance().remainingEventAndTimeOperatorMap.entrySet()) {
boolean timeout = false;
while (true) {
- if (entry.getValue().getRemainingNonHeartbeatEvents() > 0) {
+ if (entry.getValue().getRemainingNonHeartbeatEvents() == 0) {
logger.info(
"Successfully waited for pipe {} to finish.", entry.getValue().getPipeName());
break;
(iotdb) 01/11: [To dev/1.3] Fixed multiple bugs of pipe
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bf809bae0682282e212d24a5c7994ad6d5fda756
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Tue Jul 15 17:53:16 2025 +0800
[To dev/1.3] Fixed multiple bugs of pipe
---
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 2 +
.../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 6 +-
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeClusterIT.java | 6 +-
.../IoTDBPipeConnectorCompressionIT.java | 7 +-
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 1 -
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeWithLoadIT.java | 6 +-
.../pipe/it/manual/AbstractPipeDualManualIT.java | 6 +-
.../manual/IoTDBPipeTypeConversionISessionIT.java | 12 +-
.../pipe/it/manual/IoTDBPipeTypeConversionIT.java | 2 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 1 +
.../it/cluster/IoTDBSubscriptionRestartIT.java | 3 +-
.../it/dual/AbstractSubscriptionDualIT.java | 3 +
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 3 +
.../it/dual/IoTDBSubscriptionTimePrecisionIT.java | 2 +
.../it/dual/IoTDBSubscriptionTopicIT.java | 3 +
.../it/local/AbstractSubscriptionLocalIT.java | 7 +-
.../it/triple/AbstractSubscriptionTripleIT.java | 4 +
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 6 +-
.../event/dml/insertion/TsFileInsertionEvent.java | 10 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../agent/task/PipeConfigNodeSubtaskExecutor.java | 4 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 8 +
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../connector/PipeConfigNodeThriftRequestTest.java | 4 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 30 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 -
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 548 ++++++++++---------
.../task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../execution/PipeConnectorSubtaskExecutor.java | 8 +-
.../execution/PipeProcessorSubtaskExecutor.java | 2 +-
.../task/execution/PipeSubtaskExecutorManager.java | 18 +-
.../agent/task/stage/PipeTaskConnectorStage.java | 6 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 5 +
.../connector/PipeConnectorSubtaskManager.java | 19 +-
.../PipeRealtimePriorityBlockingQueue.java | 151 +++++-
.../client/IoTDBDataNodeAsyncClientManager.java | 36 ++
.../evolvable/batch/PipeTabletEventBatch.java | 39 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 10 +-
.../batch/PipeTabletEventTsFileBatch.java | 7 +
.../evolvable/request/PipeTransferPlanNodeReq.java | 1 -
.../request/PipeTransferTabletBatchReq.java | 1 -
.../request/PipeTransferTabletBinaryReq.java | 1 -
.../request/PipeTransferTabletInsertNodeReq.java | 1 -
.../request/PipeTransferTabletRawReq.java | 1 -
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +-
.../PipeConsensusTransferBatchReqBuilder.java | 3 +-
.../async/IoTDBDataRegionAsyncConnector.java | 91 ++--
.../PipeTransferTabletInsertionEventHandler.java | 2 +-
.../async/handler/PipeTransferTsFileHandler.java | 45 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 3 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../consensus/PipeConsensusSubtaskExecutor.java | 6 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 124 ++---
.../event/common/terminate/PipeTerminateEvent.java | 44 --
.../tsfile/PipeCompactedTsFileInsertionEvent.java | 252 +++++++++
.../common/tsfile/PipeTsFileInsertionEvent.java | 129 ++---
.../TsFileInsertionDataContainerProvider.java | 7 +-
.../event/realtime/PipeRealtimeEventFactory.java | 16 +-
.../dataregion/DataRegionWatermarkInjector.java | 4 -
.../dataregion/IoTDBDataRegionExtractor.java | 107 ++--
.../PipeHistoricalDataRegionTsFileExtractor.java | 146 ++---
.../realtime/PipeRealtimeDataRegionExtractor.java | 35 ++
.../PipeRealtimeDataRegionHybridExtractor.java | 180 ++-----
.../PipeRealtimeDataRegionLogExtractor.java | 5 +-
.../PipeRealtimeDataRegionTsFileExtractor.java | 7 +
.../realtime/assigner/PipeDataRegionAssigner.java | 50 +-
.../PipeTsFileEpochProgressIndexKeeper.java | 30 +-
.../dataregion/realtime/epoch/TsFileEpoch.java | 17 +-
.../realtime/epoch/TsFileEpochManager.java | 4 +-
.../listener/PipeInsertionDataNodeListener.java | 14 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 3 -
.../PipeDataNodeRemainingEventAndTimeOperator.java | 22 +-
.../overview/PipeDataNodeSinglePipeMetrics.java | 84 ++-
.../pipe/metric/overview/PipeResourceMetrics.java | 39 +-
.../overview/PipeWALInsertNodeCacheMetrics.java | 86 ---
.../sink/PipeDataRegionConnectorMetrics.java | 34 ++
.../twostage/exchange/payload/CombineRequest.java | 1 -
.../payload/FetchCombineResultRequest.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 16 +
.../pipe/resource/PipeDataNodeResourceManager.java | 25 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 33 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 12 +-
.../resource/tsfile/PipeCompactionManager.java | 68 +++
...Resource.java => PipeTsFilePublicResource.java} | 158 +-----
.../pipe/resource/tsfile/PipeTsFileResource.java | 253 +--------
.../resource/tsfile/PipeTsFileResourceManager.java | 329 ++++++------
.../db/pipe/resource/wal/PipeWALResource.java | 169 ------
.../pipe/resource/wal/PipeWALResourceManager.java | 142 -----
.../wal/hardlink/PipeWALHardlinkResource.java | 47 --
.../hardlink/PipeWALHardlinkResourceManager.java | 152 ------
.../wal/selfhost/PipeWALSelfHostResource.java | 41 --
.../selfhost/PipeWALSelfHostResourceManager.java | 38 --
.../impl/DataNodeInternalRPCServiceImpl.java | 95 +++-
.../plan/analyze/LoadTsFileAnalyzer.java | 34 +-
.../analyze/lock/DataNodeSchemaLockManager.java | 1 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 8 -
.../db/storageengine/dataregion/DataRegion.java | 5 +-
.../task/InsertionCrossSpaceCompactionTask.java | 4 +-
.../compaction/execute/utils/CompactionUtils.java | 7 +-
.../dataregion/flush/NotifyFlushMemTable.java | 7 -
.../dataregion/memtable/AbstractMemTable.java | 13 -
.../dataregion/memtable/IMemTable.java | 4 -
.../dataregion/memtable/TsFileProcessor.java | 38 +-
.../dataregion/tsfile/TsFileManager.java | 13 +
.../dataregion/tsfile/TsFileResource.java | 39 +-
.../dataregion/wal/buffer/WALBuffer.java | 6 -
.../dataregion/wal/buffer/WALEntry.java | 4 +-
.../wal/checkpoint/CheckpointManager.java | 70 +--
.../dataregion/wal/checkpoint/MemTableInfo.java | 19 +-
.../dataregion/wal/io/WALByteBufReader.java | 13 -
.../dataregion/wal/node/WALFakeNode.java | 4 +-
.../storageengine/dataregion/wal/node/WALNode.java | 96 +---
.../dataregion/wal/utils/WALEntryHandler.java | 233 --------
.../dataregion/wal/utils/WALEntryPosition.java | 229 --------
.../dataregion/wal/utils/WALInsertNodeCache.java | 335 ------------
.../wal/utils/listener/WALFlushListener.java | 12 +-
.../db/storageengine/load/LoadTsFileManager.java | 1 +
...eeStatementDataTypeConvertExecutionVisitor.java | 169 ++----
.../converter/LoadTsFileDataTypeConverter.java | 54 +-
.../execution/SubscriptionSubtaskExecutor.java | 9 +-
.../task/stage/SubscriptionTaskConnectorStage.java | 4 +-
.../connector/PipeDataNodeThriftRequestTest.java | 12 -
.../event/TsFileInsertionDataContainerTest.java | 16 +-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 101 ++--
.../resource/PipeTsFileResourceManagerTest.java | 96 ++--
.../PipeWALHardlinkResourceManagerTest.java | 114 ----
.../TsFileResourceProgressIndexTest.java | 23 +-
.../wal/checkpoint/CheckpointManagerTest.java | 172 ------
.../dataregion/wal/node/WALEntryHandlerTest.java | 586 ---------------------
.../dataregion/wal/node/WALNodeTest.java | 17 -
.../wal/node/WalDeleteOutdatedNewTest.java | 65 ---
.../wal/utils/WALInsertNodeCacheTest.java | 190 -------
.../commons/concurrent/IoTDBThreadPoolFactory.java | 15 +
.../iotdb/commons/concurrent/ThreadName.java | 10 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 397 +++++---------
.../commons/consensus/index/ProgressIndex.java | 2 +-
.../consensus/index/impl/IoTProgressIndex.java | 27 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 58 +-
.../agent/task/execution/PipeSubtaskExecutor.java | 45 +-
.../task/progress/PipeEventCommitManager.java | 25 +-
.../agent/task/progress/PipeEventCommitter.java | 12 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 5 -
.../agent/task/subtask/PipeReportableSubtask.java | 24 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 158 ++----
.../iotdb/commons/pipe/config/PipeDescriptor.java | 127 ++---
.../config/constant/PipeConnectorConstant.java | 2 +-
.../thrift/request/PipeTransferFilePieceReq.java | 1 -
.../thrift/request/PipeTransferFileSealReqV1.java | 1 -
.../thrift/request/PipeTransferFileSealReqV2.java | 1 -
.../thrift/request/PipeTransferHandshakeV1Req.java | 1 -
.../thrift/request/PipeTransferHandshakeV2Req.java | 1 -
.../thrift/request/PipeTransferSliceReq.java | 1 -
.../pipe/connector/protocol/IoTDBConnector.java | 12 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 13 +
.../commons/pipe/resource/log/PipeLogStatus.java | 2 +-
.../iotdb/commons/service/metric/enums/Metric.java | 9 +-
170 files changed, 2556 insertions(+), 5506 deletions(-)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
index fe4190d7708..a7d8918d1c9 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
@@ -429,6 +429,12 @@ public class MppCommonConfig extends MppBaseConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ setProperty("pipe_enable_memory_checked", String.valueOf(isPipeEnableMemoryCheck));
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
setProperty("pipe_air_gap_receiver_enabled", String.valueOf(isPipeAirGapReceiverEnabled));
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
index c2ade6eace0..e4a4ace7a44 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
@@ -438,6 +438,13 @@ public class MppSharedCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ dnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
+ cnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
dnConfig.setPipeAirGapReceiverEnabled(isPipeAirGapReceiverEnabled);
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
index 581c5f4345e..23a8dad317e 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
@@ -308,6 +308,11 @@ public class RemoteCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
return this;
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
index 16d50954209..178f182624b 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
@@ -138,6 +138,8 @@ public interface CommonConfig {
CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode);
+ CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck);
+
CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled);
CommonConfig setDriverTaskExecutionTimeSliceInMs(long driverTaskExecutionTimeSliceInMs);
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
index 5b4ae76332f..13131dc617f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
@@ -36,6 +36,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -901,6 +902,7 @@ public class IoTDBLoadTsFileIT {
}
@Test
+ @Ignore("Load with conversion is currently banned")
public void testLoadWithConvertOnTypeMismatch() throws Exception {
List<Pair<MeasurementSchema, MeasurementSchema>> measurementSchemas =
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
index 1817efdaa1d..59478fc36e6 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
@@ -48,13 +48,15 @@ abstract class AbstractPipeDualAutoIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
index 65eb2458df7..9da5591bb2f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
@@ -59,14 +59,16 @@ public class IoTDBPipeAutoConflictIT extends AbstractPipeDualAutoIT {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java
index d2ca8ce138c..c8d5bc0b471 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java
@@ -74,7 +74,8 @@ public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
@@ -84,7 +85,8 @@ public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT {
.setSchemaReplicationFactor(3)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java
index 97213150069..2114ed37c7f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java
@@ -66,14 +66,17 @@ public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualAutoIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
+
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setPipeAirGapReceiverEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
index 7fbb0f3a717..9a8ec2fc18e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
@@ -459,7 +459,6 @@ public class IoTDBPipeDataSinkIT extends AbstractPipeDualAutoIT {
testLoadTsFileWithoutVerify("async");
}
- @Test
private void testLoadTsFileWithoutVerify(final String loadTsFileStrategy) throws Exception {
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
index 0b4636c0d13..4b33de0a31c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
@@ -71,13 +71,15 @@ public class IoTDBPipeExtractorIT extends AbstractPipeDualAutoIT {
// Disable sender compaction for tsfile determination in loose range test
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
- .setEnableCrossSpaceCompaction(false);
+ .setEnableCrossSpaceCompaction(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java
index addf0314b40..771d50c97c6 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java
@@ -65,13 +65,15 @@ public class IoTDBPipeIdempotentIT extends AbstractPipeDualAutoIT {
// of the tested idempotent sql.
.setDefaultSchemaRegionGroupNumPerDatabase(1)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java
index 125fd6972b1..13a63a585a8 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java
@@ -59,13 +59,15 @@ public class IoTDBPipeProcessorIT extends AbstractPipeDualAutoIT {
.setAutoCreateSchemaEnabled(true)
.setTimestampPrecision("ms")
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java
index c5d41f001cb..1f646768525 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java
@@ -74,7 +74,8 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualAutoIT {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
- .setDataReplicationFactor(dataRegionReplicationFactor);
+ .setDataReplicationFactor(dataRegionReplicationFactor)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
@@ -83,7 +84,8 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualAutoIT {
.setSchemaRegionConsensusProtocolClass(schemaRegionConsensus)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
- .setDataReplicationFactor(dataRegionReplicationFactor);
+ .setDataReplicationFactor(dataRegionReplicationFactor)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java
index 563377c927e..c71e212b629 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java
@@ -61,13 +61,15 @@ public class IoTDBPipeWithLoadIT extends AbstractPipeDualAutoIT {
// Disable sender compaction to test mods
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
- .setEnableCrossSpaceCompaction(false);
+ .setEnableCrossSpaceCompaction(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java
index 3e9f5111165..b7091a1db32 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java
@@ -48,13 +48,15 @@ abstract class AbstractPipeDualManualIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java
index 9af757acbde..fc2b32e145b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java
@@ -44,6 +44,7 @@ import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -76,6 +77,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertTabletReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -94,6 +96,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedTabletReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -103,6 +106,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertRecordsReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -137,6 +141,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertRecordReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -177,6 +182,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedRecordReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -225,6 +231,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedRecordsReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -253,6 +260,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertStringRecordsOfOneDeviceReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -281,6 +289,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedStringRecordsOfOneDeviceReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -381,13 +390,12 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT
String sql =
String.format(
"create pipe test%s"
- + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='%s','realtime.enable'='%s','history.enable'='%s')"
+ + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='%s','realtime.enable'='%s','history.enable'='true')"
+ " with processor ('processor'='do-nothing-processor')"
+ " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')",
diff,
isTSFile ? "file" : "forced-log",
!isTSFile,
- isTSFile,
receiverEnv.getIP(),
receiverEnv.getPort(),
isTSFile ? "tsfile" : "tablet");
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java
index 4ebce6b978c..24b40d2e0c9 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java
@@ -234,7 +234,7 @@ public class IoTDBPipeTypeConversionIT extends AbstractPipeDualManualIT {
String sql =
String.format(
"create pipe test"
- + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='forced-log','realtime.enable'='true','history.enable'='false')"
+ + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='forced-log')"
+ " with processor ('processor'='do-nothing-processor')"
+ " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='tablet')",
receiverEnv.getIP(), receiverEnv.getPort());
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
index 4c513576488..9a484426d76 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
@@ -36,6 +36,7 @@ abstract class AbstractPipeSingleIT {
env.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
// 10 min, assert that the operations will not time out
env.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+ env.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
env.initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
index de17dec1ed4..8156139b84c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
@@ -82,7 +82,8 @@ public class IoTDBSubscriptionRestartIT extends AbstractSubscriptionIT {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(3)
- .setDataReplicationFactor(2);
+ .setDataReplicationFactor(2)
+ .setIsPipeEnableMemoryCheck(false);
EnvFactory.getEnv().initClusterEnvironment(3, 3);
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
index 594f9efe691..5e61607ff32 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
@@ -58,6 +58,9 @@ public abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java
index 6f8cba43e79..5ad46b10977 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java
@@ -117,8 +117,11 @@ public class IoTDBSubscriptionConsumerGroupIT extends AbstractSubscriptionDualIT
protected void setUpConfig() {
super.setUpConfig();
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+
// Enable air gap receiver
receiverEnv.getConfig().getCommonConfig().setPipeAirGapReceiverEnabled(true);
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java
index 4327b7a00f0..1939a986c68 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java
@@ -68,7 +68,9 @@ public class IoTDBSubscriptionTimePrecisionIT extends AbstractSubscriptionDualIT
// Set timestamp precision to nanosecond
senderEnv.getConfig().getCommonConfig().setTimestampPrecision("ns");
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
receiverEnv.getConfig().getCommonConfig().setTimestampPrecision("ns");
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Test
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java
index b046f09a6da..37971833b97 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java
@@ -91,6 +91,9 @@ public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT {
.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30);
senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerInitialSyncDelayMinutes(1);
senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerSyncIntervalMinutes(1);
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Test
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
index 3f415051928..7b564023498 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
@@ -33,7 +33,12 @@ public abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT
super.setUp();
// enable subscription
- EnvFactory.getEnv().getConfig().getCommonConfig().setSubscriptionEnabled(true);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSubscriptionEnabled(true)
+ .setIsPipeEnableMemoryCheck(false);
+
EnvFactory.getEnv().initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
index 3b49eb8ed13..1e473d4fd44 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
@@ -69,6 +69,10 @@ public abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionI
sender.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiver1.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiver2.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+ // reduce tsfile batch memory usage
+ sender.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiver1.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiver2.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
index 6ad0c843e27..115dafbc5dc 100644
--- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
@@ -53,7 +53,11 @@ public class ExportTsFileTestIT extends AbstractScriptIT {
@BeforeClass
public static void setUp() throws Exception {
// enable subscription
- EnvFactory.getEnv().getConfig().getCommonConfig().setSubscriptionEnabled(true);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSubscriptionEnabled(true)
+ .setIsPipeEnableMemoryCheck(false);
EnvFactory.getEnv().initClusterEnvironment();
ip = EnvFactory.getEnv().getIP();
diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
index 3d041867242..4c7fffcfba5 100644
--- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
+++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.pipe.api.event.dml.insertion;
import org.apache.iotdb.pipe.api.event.Event;
+import java.io.File;
+
/**
* {@link TsFileInsertionEvent} is used to define the event of writing TsFile. Event data stores in
* disks, which is compressed and encoded, and requires IO cost for computational processing.
@@ -34,4 +36,12 @@ public interface TsFileInsertionEvent extends Event, AutoCloseable {
* @return {@code Iterable<TabletInsertionEvent>} the list of {@link TabletInsertionEvent}
*/
Iterable<TabletInsertionEvent> toTabletInsertionEvents();
+
+ /**
+ * Get the file that stores the data of this {@link TsFileInsertionEvent}. The file is compressed
+ * and encoded, and requires IO cost for computational processing.
+ *
+ * @return the file that stores the data of this {@link TsFileInsertionEvent}
+ */
+ File getTsFile();
}
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index fe9b40de33f..e2ab4b0ff43 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -248,6 +248,7 @@ public enum TSStatusCode {
PIPE_RECEIVER_USER_CONFLICT_EXCEPTION(1810),
PIPE_CONFIG_RECEIVER_HANDSHAKE_NEEDED(1811),
PIPE_TRANSFER_SLICE_OUT_OF_ORDER(1812),
+ PIPE_PUSH_META_TIMEOUT(1813),
// Subscription
SUBSCRIPTION_VERSION_ERROR(1900),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java
index 9aea3edfa38..1ac64eb246f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java
@@ -28,7 +28,7 @@ public class PipeConfigNodeSubtaskExecutor extends PipeSubtaskExecutor {
private static final int THREAD_NUM = 1;
private PipeConfigNodeSubtaskExecutor() {
- super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL, true);
+ super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL.getName(), true);
}
/**
@@ -36,7 +36,7 @@ public class PipeConfigNodeSubtaskExecutor extends PipeSubtaskExecutor {
*/
@TestOnly
public PipeConfigNodeSubtaskExecutor(final Object ignored) {
- super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL, true);
+ super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL.getName(), true);
}
private static class PipeSchemaSubtaskExecutorHolder {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
index b115ba25b4a..911909d104c 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
@@ -46,11 +46,13 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
@@ -250,4 +252,10 @@ public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
resp.setPipeRemainingEventCountList(pipeRemainingEventCountList);
resp.setPipeRemainingTimeList(pipeRemainingTimeList);
}
+
+ @Override
+ public void runPipeTasks(
+ final Collection<PipeTask> pipeTasks, final Consumer<PipeTask> runSingle) {
+ pipeTasks.forEach(runSingle);
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
index 3ab75270b0b..eb4ffc48c79 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
@@ -422,12 +422,20 @@ public abstract class AbstractOperatePipeProcedureV2
* @return Error messages for the given pipe after pushing pipe meta
*/
public static String parsePushPipeMetaExceptionForPipe(
- String pipeName, Map<Integer, TPushPipeMetaResp> respMap) {
+ final String pipeName, final Map<Integer, TPushPipeMetaResp> respMap) {
final StringBuilder exceptionMessageBuilder = new StringBuilder();
- for (Map.Entry<Integer, TPushPipeMetaResp> respEntry : respMap.entrySet()) {
- int dataNodeId = respEntry.getKey();
- TPushPipeMetaResp resp = respEntry.getValue();
+ for (final Map.Entry<Integer, TPushPipeMetaResp> respEntry : respMap.entrySet()) {
+ final int dataNodeId = respEntry.getKey();
+ final TPushPipeMetaResp resp = respEntry.getValue();
+
+ if (resp.getStatus().getCode() == TSStatusCode.PIPE_PUSH_META_TIMEOUT.getStatusCode()) {
+ exceptionMessageBuilder.append(
+ String.format(
+ "DataNodeId: %s, Message: Timeout to wait for lock while processing pushPipeMeta on dataNodes.",
+ dataNodeId));
+ continue;
+ }
if (resp.getStatus().getCode() == TSStatusCode.PIPE_PUSH_META_ERROR.getStatusCode()) {
if (!resp.isSetExceptionMessages()) {
@@ -438,7 +446,7 @@ public abstract class AbstractOperatePipeProcedureV2
continue;
}
- AtomicBoolean hasException = new AtomicBoolean(false);
+ final AtomicBoolean hasException = new AtomicBoolean(false);
resp.getExceptionMessages()
.forEach(
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
index c2c04f94529..8854ee647f9 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
@@ -44,7 +44,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getTimestampPrecision(), deserializeReq.getTimestampPrecision());
}
@@ -57,7 +56,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
}
@Test
@@ -72,7 +70,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileName(), deserializeReq.getFileName());
Assert.assertEquals(req.getStartWritingOffset(), deserializeReq.getStartWritingOffset());
@@ -95,7 +92,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileNames(), deserializeReq.getFileNames());
Assert.assertEquals(req.getFileLengths(), deserializeReq.getFileLengths());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 446a428eee7..890af218a59 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -314,6 +314,8 @@ public class IoTDBConfig {
private String extPipeDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_PIPE_FOLDER_NAME;
+ private int pipeTaskThreadCount = 5;
+
/** External lib directory for MQTT, stores user-uploaded JAR files */
private String mqttDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.MQTT_FOLDER_NAME;
@@ -1046,6 +1048,8 @@ public class IoTDBConfig {
/** Policy of DataNodeSchemaCache eviction */
private String dataNodeSchemaCacheEvictionPolicy = "FIFO";
+ private int schemaThreadCount = 5;
+
private String readConsistencyLevel = "strong";
/** Maximum execution time of a DriverTask */
@@ -1147,8 +1151,6 @@ public class IoTDBConfig {
private long loadTsFileTabletConversionBatchMemorySizeInBytes = 4096 * 1024;
- private int loadTsFileTabletConversionThreadCount = 5;
-
private long loadChunkMetadataMemorySizeInBytes = 33554432; // 32MB
private long loadMemoryAllocateRetryIntervalMs = 1000L;
@@ -3345,6 +3347,14 @@ public class IoTDBConfig {
this.extPipeDir = extPipeDir;
}
+ public int getPipeTaskThreadCount() {
+ return pipeTaskThreadCount;
+ }
+
+ public void setPipeTaskThreadCount(int pipeTaskThreadCount) {
+ this.pipeTaskThreadCount = pipeTaskThreadCount;
+ }
+
public void setPartitionCacheSize(int partitionCacheSize) {
this.partitionCacheSize = partitionCacheSize;
}
@@ -3481,6 +3491,14 @@ public class IoTDBConfig {
this.dataNodeSchemaCacheEvictionPolicy = dataNodeSchemaCacheEvictionPolicy;
}
+ public int getSchemaThreadCount() {
+ return schemaThreadCount;
+ }
+
+ public void setSchemaThreadCount(int schemaThreadCount) {
+ this.schemaThreadCount = schemaThreadCount;
+ }
+
public String getReadConsistencyLevel() {
return readConsistencyLevel;
}
@@ -4026,14 +4044,6 @@ public class IoTDBConfig {
loadTsFileTabletConversionBatchMemorySizeInBytes;
}
- public int getLoadTsFileTabletConversionThreadCount() {
- return loadTsFileTabletConversionThreadCount;
- }
-
- public void setLoadTsFileTabletConversionThreadCount(int loadTsFileTabletConversionThreadCount) {
- this.loadTsFileTabletConversionThreadCount = loadTsFileTabletConversionThreadCount;
- }
-
public long getLoadChunkMetadataMemorySizeInBytes() {
return loadChunkMetadataMemorySizeInBytes;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 25f3095d9a0..7fbd9ce2ed7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -955,6 +955,10 @@ public class IoTDBDescriptor {
}
conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir()).trim());
+ conf.setPipeTaskThreadCount(
+ Integer.parseInt(
+ properties.getProperty(
+ "pipe_task_thread_count", Integer.toString(conf.getPipeTaskThreadCount()).trim())));
// At the same time, set TSFileConfig
List<FSType> fsTypes = new ArrayList<>();
@@ -1091,6 +1095,11 @@ public class IoTDBDescriptor {
properties.getProperty(
"datanode_schema_cache_eviction_policy", conf.getDataNodeSchemaCacheEvictionPolicy()));
+ conf.setSchemaThreadCount(
+ Integer.parseInt(
+ properties.getProperty(
+ "schema_thread_count", Integer.toString(conf.getSchemaThreadCount()))));
+
loadIoTConsensusProps(properties);
loadPipeConsensusProps(properties);
@@ -2386,11 +2395,6 @@ public class IoTDBDescriptor {
properties.getProperty(
"load_tsfile_tablet_conversion_batch_memory_size_in_bytes",
String.valueOf(conf.getLoadTsFileTabletConversionBatchMemorySizeInBytes()))));
- conf.setLoadTsFileTabletConversionThreadCount(
- Integer.parseInt(
- properties.getProperty(
- "load_tsfile_tablet_conversion_thread_count",
- String.valueOf(conf.getLoadTsFileTabletConversionThreadCount()))));
conf.setLoadChunkMetadataMemorySizeInBytes(
Long.parseLong(
Optional.ofNullable(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
index 96e2e4e7299..089d4e2fda7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
@@ -36,7 +36,6 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
import org.apache.iotdb.db.service.ResourcesInformationHolder;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
import org.apache.iotdb.pipe.api.exception.PipeException;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -161,7 +160,6 @@ class PipeAgentLauncher {
try (final ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
final TGetAllPipeInfoResp getAllPipeInfoResp = configNodeClient.getAllPipeInfo();
- WALInsertNodeCache.init();
PipeTabletEventBatch.init();
if (getAllPipeInfoResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
throw new StartupException("Failed to get pipe task meta from config node.");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
index 5a1f9d8b845..afb05fc2c39 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
@@ -35,7 +35,6 @@ import org.apache.iotdb.commons.service.ServiceType;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningQueue;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
@@ -86,15 +85,6 @@ public class PipeDataNodeRuntimeAgent implements IService {
PipeConfig.getInstance().printAllConfigs();
PipeAgentLauncher.launchPipeTaskAgent();
- registerPeriodicalJob(
- "PipeTaskAgent#restartAllStuckPipes",
- PipeDataNodeAgent.task()::restartAllStuckPipes,
- PipeConfig.getInstance().getPipeStuckRestartIntervalSeconds());
- registerPeriodicalJob(
- "PipeTaskAgent#flushDataRegionIfNeeded",
- PipeTerminateEvent::flushDataRegionIfNeeded,
- PipeConfig.getInstance().getPipeFlushAfterLastTerminateSeconds());
-
pipePeriodicalJobExecutor.start();
if (PipeConfig.getInstance().getPipeEventReferenceTrackingEnabled()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
index 1513d8ebdeb..15e20347393 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
@@ -21,6 +21,10 @@ package org.apache.iotdb.db.pipe.agent.task;
import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.consensus.SchemaRegionId;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
@@ -36,8 +40,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeType;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant;
import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
-import org.apache.iotdb.commons.service.metric.MetricService;
-import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -47,12 +50,10 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.agent.task.builder.PipeDataNodeBuilder;
import org.apache.iotdb.db.pipe.agent.task.builder.PipeDataNodeTaskBuilder;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
-import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningFilter;
import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
-import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
@@ -62,10 +63,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.SystemMetric;
import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp;
import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaRespExceptionMessage;
@@ -83,6 +81,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -91,21 +90,52 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_START_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATH_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATTERN_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_START_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_START_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_PATH_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_PATTERN_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_START_TIME_KEY;
+
public class PipeDataNodeTaskAgent extends PipeTaskAgent {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeDataNodeTaskAgent.class);
protected static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final AtomicLong LAST_FORCED_RESTART_TIME =
- new AtomicLong(System.currentTimeMillis());
- private static final Map<String, AtomicLong> PIPE_NAME_TO_LAST_RESTART_TIME_MAP =
- new ConcurrentHashMap<>();
+ private final ExecutorService pipeExecutor =
+ new WrappedThreadPoolExecutor(
+ 0,
+ IoTDBDescriptor.getInstance().getConfig().getPipeTaskThreadCount(),
+ 0L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount()),
+ new IoTThreadFactory(ThreadName.PIPE_PARALLEL_EXECUTION_POOL.getName()),
+ ThreadName.PIPE_PARALLEL_EXECUTION_POOL.getName(),
+ new ThreadPoolExecutor.CallerRunsPolicy());
////////////////////////// Pipe Task Management Entry //////////////////////////
@@ -120,54 +150,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
return new PipeDataNodeBuilder(pipeMetaFromConfigNode).build();
}
- ////////////////////////// Manage by Pipe Name //////////////////////////
-
- @Override
- protected void startPipe(final String pipeName, final long creationTime) {
- final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName);
- final PipeStatus status = existedPipeMeta.getRuntimeMeta().getStatus().get();
- if (PipeStatus.STOPPED.equals(status) || status == null) {
- restartPipeToReloadResourceIfNeeded(existedPipeMeta);
- }
-
- super.startPipe(pipeName, creationTime);
- }
-
- private void restartPipeToReloadResourceIfNeeded(final PipeMeta pipeMeta) {
- if (System.currentTimeMillis() - pipeMeta.getStaticMeta().getCreationTime()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()) {
- return;
- }
-
- final AtomicLong lastRestartTime =
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP.get(pipeMeta.getStaticMeta().getPipeName());
- if (lastRestartTime != null
- && System.currentTimeMillis() - lastRestartTime.get()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()) {
- LOGGER.info(
- "Skipping reload resource for stopped pipe {} before starting it because reloading resource is too frequent.",
- pipeMeta.getStaticMeta().getPipeName());
- return;
- }
-
- if (PIPE_NAME_TO_LAST_RESTART_TIME_MAP.isEmpty()) {
- LOGGER.info(
- "Flushing storage engine before restarting pipe {}.",
- pipeMeta.getStaticMeta().getPipeName());
- final long currentTime = System.currentTimeMillis();
- StorageEngine.getInstance().syncCloseAllProcessor();
- WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes();
- LOGGER.info(
- "Finished flushing storage engine, time cost: {} ms.",
- System.currentTimeMillis() - currentTime);
- }
-
- restartStuckPipe(pipeMeta);
- LOGGER.info(
- "Reloaded resource for stopped pipe {} before starting it.",
- pipeMeta.getStaticMeta().getPipeName());
- }
-
///////////////////////// Manage by regionGroupId /////////////////////////
@Override
@@ -383,9 +365,8 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
///////////////////////// Heartbeat /////////////////////////
public void collectPipeMetaList(final TDataNodeHeartbeatResp resp) throws TException {
- // Try the lock instead of directly acquire it to prevent the block of the cluster heartbeat
- // 10s is the half of the HEARTBEAT_TIMEOUT_TIME defined in class BaseNodeCache in ConfigNode
- if (!tryReadLockWithTimeOut(10)) {
+ if (!tryReadLockWithTimeOut(
+ CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS() * 2L / 3)) {
return;
}
try {
@@ -558,226 +539,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
PipeInsertionDataNodeListener.getInstance().listenToHeartbeat(true);
}
- ///////////////////////// Restart Logic /////////////////////////
-
- public void restartAllStuckPipes() {
- final List<String> removedPipeName = removeOutdatedPipeInfoFromLastRestartTimeMap();
- if (!removedPipeName.isEmpty()) {
- final long currentTime = System.currentTimeMillis();
- LOGGER.info(
- "Pipes {} now can dynamically adjust their extraction strategies. "
- + "Start to flush storage engine to trigger the adjustment.",
- removedPipeName);
- StorageEngine.getInstance().syncCloseAllProcessor();
- WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes();
- LOGGER.info(
- "Finished flushing storage engine, time cost: {} ms.",
- System.currentTimeMillis() - currentTime);
- LOGGER.info("Skipping restarting pipes this round because of the dynamic flushing.");
- return;
- }
-
- if (!tryWriteLockWithTimeOut(5)) {
- return;
- }
-
- final Set<PipeMeta> stuckPipes;
- try {
- stuckPipes = findAllStuckPipes();
- } finally {
- releaseWriteLock();
- }
-
- // If the pipe has been restarted recently, skip it.
- stuckPipes.removeIf(
- pipeMeta -> {
- final AtomicLong lastRestartTime =
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP.get(pipeMeta.getStaticMeta().getPipeName());
- return lastRestartTime != null
- && System.currentTimeMillis() - lastRestartTime.get()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs();
- });
-
- // Restart all stuck pipes.
- // Note that parallelStream cannot be used here. The method PipeTaskAgent#dropPipe also uses
- // parallelStream. If parallelStream is used here, the subtasks generated inside the dropPipe
- // may not be scheduled by the worker thread of ForkJoinPool because of less available threads,
- // and the parent task will wait for the completion of the subtasks in ForkJoinPool forever,
- // causing the deadlock.
- stuckPipes.forEach(this::restartStuckPipe);
- }
-
- private List<String> removeOutdatedPipeInfoFromLastRestartTimeMap() {
- final List<String> removedPipeName = new ArrayList<>();
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP
- .entrySet()
- .removeIf(
- entry -> {
- final AtomicLong lastRestartTime = entry.getValue();
- final boolean shouldRemove =
- lastRestartTime == null
- || PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()
- <= System.currentTimeMillis() - lastRestartTime.get();
- if (shouldRemove) {
- removedPipeName.add(entry.getKey());
- }
- return shouldRemove;
- });
- return removedPipeName;
- }
-
- private Set<PipeMeta> findAllStuckPipes() {
- final Set<PipeMeta> stuckPipes = new HashSet<>();
-
- if (System.currentTimeMillis() - LAST_FORCED_RESTART_TIME.get()
- > PipeConfig.getInstance().getPipeSubtaskExecutorForcedRestartIntervalMs()) {
- LAST_FORCED_RESTART_TIME.set(System.currentTimeMillis());
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- stuckPipes.add(pipeMeta);
- }
- if (!stuckPipes.isEmpty()) {
- LOGGER.warn(
- "All {} pipe(s) will be restarted because of forced restart policy.",
- stuckPipes.size());
- }
- return stuckPipes;
- }
-
- final long totalLinkedButDeletedTsFileResourceRamSize =
- PipeDataNodeResourceManager.tsfile().getTotalLinkedButDeletedTsFileResourceRamSize();
- final long totalInsertNodeFloatingMemoryUsageInBytes = getAllFloatingMemoryUsageInByte();
- final long totalFloatingMemorySizeInBytes =
- PipeMemoryManager.getTotalFloatingMemorySizeInBytes();
- if (totalInsertNodeFloatingMemoryUsageInBytes + totalLinkedButDeletedTsFileResourceRamSize
- >= totalFloatingMemorySizeInBytes) {
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- stuckPipes.add(pipeMeta);
- }
- if (!stuckPipes.isEmpty()) {
- LOGGER.warn(
- "All {} pipe(s) will be restarted because linked but deleted tsFiles' resource size {} and all insertNode's size {} exceeds limit {}.",
- stuckPipes.size(),
- totalLinkedButDeletedTsFileResourceRamSize,
- totalInsertNodeFloatingMemoryUsageInBytes,
- totalFloatingMemorySizeInBytes);
- }
- return stuckPipes;
- }
-
- final Map<String, IoTDBDataRegionExtractor> taskId2ExtractorMap =
- PipeDataRegionExtractorMetrics.getInstance().getExtractorMap();
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- final String pipeName = pipeMeta.getStaticMeta().getPipeName();
- final List<IoTDBDataRegionExtractor> extractors =
- taskId2ExtractorMap.values().stream()
- .filter(e -> e.getPipeName().equals(pipeName) && e.shouldExtractInsertion())
- .collect(Collectors.toList());
-
- if (extractors.isEmpty()) {
- continue;
- }
-
- // Extractors of this pipe might not pin too much MemTables,
- // still need to check if linked-and-deleted TsFile count exceeds limit.
- // Typically, if deleted tsFiles are too abundant all pipes may need to restart.
- if ((CONFIG.isEnableSeqSpaceCompaction()
- || CONFIG.isEnableUnseqSpaceCompaction()
- || CONFIG.isEnableCrossSpaceCompaction())
- && mayDeletedTsFileSizeReachDangerousThreshold()) {
- LOGGER.warn(
- "Pipe {} needs to restart because too many TsFiles are out-of-date.",
- pipeMeta.getStaticMeta());
- stuckPipes.add(pipeMeta);
- continue;
- }
-
- // Try to restart the stream mode pipes for releasing memTables.
- if (extractors.get(0).isStreamMode()) {
- if (extractors.stream().anyMatch(IoTDBDataRegionExtractor::hasConsumedAllHistoricalTsFiles)
- && (mayMemTablePinnedCountReachDangerousThreshold()
- || mayWalSizeReachThrottleThreshold())) {
- // Extractors of this pipe may be stuck and is pinning too many MemTables.
- LOGGER.warn(
- "Pipe {} needs to restart because too many memTables are pinned or the WAL size is too large. mayMemTablePinnedCountReachDangerousThreshold: {}, mayWalSizeReachThrottleThreshold: {}",
- pipeMeta.getStaticMeta(),
- mayMemTablePinnedCountReachDangerousThreshold(),
- mayWalSizeReachThrottleThreshold());
- stuckPipes.add(pipeMeta);
- }
- }
- }
-
- return stuckPipes;
- }
-
- private boolean mayDeletedTsFileSizeReachDangerousThreshold() {
- try {
- final long linkedButDeletedTsFileSize =
- PipeDataNodeResourceManager.tsfile().getTotalLinkedButDeletedTsfileSize();
- final double totalDisk =
- MetricService.getInstance()
- .getAutoGauge(
- SystemMetric.SYS_DISK_TOTAL_SPACE.toString(),
- MetricLevel.CORE,
- Tag.NAME.toString(),
- // This "system" should stay the same with the one in
- // DataNodeInternalRPCServiceImpl.
- "system")
- .getValue();
- return linkedButDeletedTsFileSize > 0
- && totalDisk > 0
- && linkedButDeletedTsFileSize
- > PipeConfig.getInstance().getPipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage()
- * totalDisk;
- } catch (final Exception e) {
- LOGGER.warn("Failed to judge if deleted TsFile size reaches dangerous threshold.", e);
- return false;
- }
- }
-
- private boolean mayMemTablePinnedCountReachDangerousThreshold() {
- return PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount() != Integer.MAX_VALUE
- && PipeDataNodeResourceManager.wal().getPinnedWalCount()
- >= 5
- * PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber();
- }
-
- private boolean mayWalSizeReachThrottleThreshold() {
- return 3 * WALManager.getInstance().getTotalDiskUsage() > 2 * CONFIG.getThrottleThreshold();
- }
-
- private void restartStuckPipe(final PipeMeta pipeMeta) {
- LOGGER.warn(
- "Pipe {} will be restarted because it is stuck or has encountered issues such as data backlog or being stopped for too long.",
- pipeMeta.getStaticMeta());
- acquireWriteLock();
- try {
- final long startTime = System.currentTimeMillis();
- final PipeMeta originalPipeMeta = pipeMeta.deepCopy4TaskAgent();
- handleDropPipe(pipeMeta.getStaticMeta().getPipeName());
-
- final long restartTime = System.currentTimeMillis();
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP
- .computeIfAbsent(pipeMeta.getStaticMeta().getPipeName(), k -> new AtomicLong(restartTime))
- .set(restartTime);
- handleSinglePipeMetaChanges(originalPipeMeta);
-
- LOGGER.warn(
- "Pipe {} was restarted because of stuck or data backlog, time cost: {} ms.",
- originalPipeMeta.getStaticMeta(),
- System.currentTimeMillis() - startTime);
- } catch (final Exception e) {
- LOGGER.warn("Failed to restart stuck pipe {}.", pipeMeta.getStaticMeta(), e);
- } finally {
- releaseWriteLock();
- }
- }
-
- public boolean isPipeTaskCurrentlyRestarted(final String pipeName) {
- return PIPE_NAME_TO_LAST_RESTART_TIME_MAP.containsKey(pipeName);
- }
-
///////////////////////// Terminate Logic /////////////////////////
public void markCompleted(final String pipeName, final int regionId) {
@@ -821,6 +582,26 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
}
}
+ @Override
+ public void runPipeTasks(
+ final Collection<PipeTask> pipeTasks, final Consumer<PipeTask> runSingle) {
+ final Set<Future<?>> pipeFuture = new HashSet<>();
+
+ pipeTasks.forEach(
+ pipeTask -> pipeFuture.add(pipeExecutor.submit(() -> runSingle.accept(pipeTask))));
+
+ for (final Future<?> future : pipeFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing pipe task: ", e);
+ throw new PipeException(e.toString());
+ }
+ }
+ }
+
+ ///////////////////////// Shutdown Logic /////////////////////////
+
public void persistAllProgressIndex() {
try (final ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
@@ -880,4 +661,203 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
releaseReadLock();
}
}
+
+ @Override
+ protected void calculateMemoryUsage(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+ if (!PipeConfig.getInstance().isPipeEnableMemoryCheck()) {
+ return;
+ }
+
+ calculateInsertNodeQueueMemory(extractorParameters, processorParameters, connectorParameters);
+
+ long needMemory = 0;
+
+ needMemory +=
+ calculateTsFileParserMemory(extractorParameters, processorParameters, connectorParameters);
+ needMemory +=
+ calculateSinkBatchMemory(extractorParameters, processorParameters, connectorParameters);
+ needMemory +=
+ calculateSendTsFileReadBufferMemory(
+ extractorParameters, processorParameters, connectorParameters);
+
+ PipeMemoryManager pipeMemoryManager = PipeDataNodeResourceManager.memory();
+ final long freeMemorySizeInBytes = pipeMemoryManager.getFreeMemorySizeInBytes();
+ final long reservedMemorySizeInBytes =
+ (long)
+ (PipeMemoryManager.getTotalMemorySizeInBytes()
+ * PipeConfig.getInstance().getReservedMemoryPercentage());
+ if (freeMemorySizeInBytes < needMemory + reservedMemorySizeInBytes) {
+ final String message =
+ String.format(
+ "Not enough memory for pipe. Need memory: %d bytes, free memory: %d bytes, reserved memory: %d bytes, total memory: %d bytes",
+ needMemory,
+ freeMemorySizeInBytes,
+ freeMemorySizeInBytes,
+ PipeMemoryManager.getTotalMemorySizeInBytes());
+ LOGGER.warn(message);
+ throw new PipeException(message);
+ }
+ }
+
+ private void calculateInsertNodeQueueMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // Realtime extractor is enabled by default, so we only need to check the source realtime
+ if (!extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
+ EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)) {
+ return;
+ }
+
+ // If the realtime mode is batch or file, we do not need to allocate memory
+ final String realtimeMode =
+ extractorParameters.getStringByKeys(
+ PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_KEY,
+ PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY);
+ if (PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE.equals(realtimeMode)
+ || PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FILE_VALUE.equals(realtimeMode)) {
+ return;
+ }
+
+ final long allocatedMemorySizeInBytes = this.getAllFloatingMemoryUsageInByte();
+ final long remainingMemory =
+ PipeMemoryManager.getTotalFloatingMemorySizeInBytes() - allocatedMemorySizeInBytes;
+ if (remainingMemory < PipeConfig.getInstance().PipeInsertNodeQueueMemory()) {
+ final String message =
+ String.format(
+ "Not enough memory for pipe. Need Floating memory: %d bytes, free Floating memory: %d bytes",
+ PipeConfig.getInstance().PipeInsertNodeQueueMemory(), remainingMemory);
+ LOGGER.warn(message);
+ throw new PipeException(message);
+ }
+ }
+
+ private long calculateTsFileParserMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // If the extractor is not history, we do not need to allocate memory
+ boolean isExtractorHistory =
+ extractorParameters.getBooleanOrDefault(
+ SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE);
+
+ // If the extractor is history, and has start/end time, we need to allocate memory
+ boolean isTSFileParser =
+ isExtractorHistory
+ && extractorParameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY);
+
+ isTSFileParser =
+ isTSFileParser
+ || (isExtractorHistory
+ && extractorParameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY));
+
+ // if the extractor has start/end time, we need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(
+ SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY);
+
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY);
+
+ // If the extractor has pattern or path, we need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY);
+
+ isTSFileParser =
+ isTSFileParser || extractorParameters.hasAnyAttributes(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY);
+
+ // If the extractor is not hybrid, we do need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || !PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals(
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY,
+ PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE));
+
+ if (!isTSFileParser) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getTsFileParserMemory();
+ }
+
+ private long calculateSinkBatchMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // If the connector format is tsfile , we need to use batch
+ boolean needUseBatch =
+ PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE.equals(
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY,
+ PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE));
+
+ if (needUseBatch) {
+ return PipeConfig.getInstance().getSinkBatchMemoryTsFile();
+ }
+
+ // If the connector is batch mode, we need to use batch
+ needUseBatch =
+ connectorParameters.getBooleanOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY,
+ PipeConnectorConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY),
+ PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE);
+
+ if (!needUseBatch) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getSinkBatchMemoryInsertNode();
+ }
+
+ private long calculateSendTsFileReadBufferMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+ // If the extractor is history enable, we need to transfer tsfile
+ boolean needTransferTsFile =
+ extractorParameters.getBooleanOrDefault(
+ SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE);
+
+ String format =
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY, PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE);
+
+ // If the connector format is tsfile and hybrid, we need to transfer tsfile
+ needTransferTsFile =
+ needTransferTsFile
+ || PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals(format)
+ || PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE.equals(format);
+
+ if (!needTransferTsFile) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getSendTsFileReadBuffer();
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
index 5e7901cb7c7..f84cb73fd69 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeType;
import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTask;
-import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeProcessorSubtaskExecutor;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeSubtaskExecutorManager;
import org.apache.iotdb.db.pipe.agent.task.stage.PipeTaskConnectorStage;
@@ -42,7 +41,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.Map;
@@ -64,19 +62,8 @@ public class PipeDataNodeTaskBuilder {
private final int regionId;
private final PipeTaskMeta pipeTaskMeta;
- private static final PipeProcessorSubtaskExecutor PROCESSOR_EXECUTOR;
- private static final Map<PipeType, PipeConnectorSubtaskExecutor> CONNECTOR_EXECUTOR_MAP;
-
- static {
- PROCESSOR_EXECUTOR = PipeSubtaskExecutorManager.getInstance().getProcessorExecutor();
- CONNECTOR_EXECUTOR_MAP = new EnumMap<>(PipeType.class);
- CONNECTOR_EXECUTOR_MAP.put(
- PipeType.USER, PipeSubtaskExecutorManager.getInstance().getConnectorExecutor());
- CONNECTOR_EXECUTOR_MAP.put(
- PipeType.SUBSCRIPTION, PipeSubtaskExecutorManager.getInstance().getSubscriptionExecutor());
- CONNECTOR_EXECUTOR_MAP.put(
- PipeType.CONSENSUS, PipeSubtaskExecutorManager.getInstance().getConsensusExecutor());
- }
+ private static final PipeProcessorSubtaskExecutor PROCESSOR_EXECUTOR =
+ PipeSubtaskExecutorManager.getInstance().getProcessorExecutor();
protected final Map<String, String> systemParameters = new HashMap<>();
@@ -117,7 +104,7 @@ public class PipeDataNodeTaskBuilder {
pipeStaticMeta.getCreationTime(),
connectorParameters,
regionId,
- CONNECTOR_EXECUTOR_MAP.get(pipeType));
+ PipeSubtaskExecutorManager.getInstance().getSubscriptionExecutor());
} else { // user pipe or consensus pipe
connectorStage =
new PipeTaskConnectorStage(
@@ -125,7 +112,9 @@ public class PipeDataNodeTaskBuilder {
pipeStaticMeta.getCreationTime(),
connectorParameters,
regionId,
- CONNECTOR_EXECUTOR_MAP.get(pipeType));
+ pipeType.equals(PipeType.USER)
+ ? PipeSubtaskExecutorManager.getInstance().getConnectorExecutorSupplier()
+ : PipeSubtaskExecutorManager.getInstance().getConsensusExecutorSupplier());
}
// The processor connects the extractor and connector.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java
index fcb4888dbde..0850c81a59d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java
@@ -23,16 +23,20 @@ import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.pipe.agent.task.execution.PipeSubtaskExecutor;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import java.util.concurrent.atomic.AtomicInteger;
+
public class PipeConnectorSubtaskExecutor extends PipeSubtaskExecutor {
+ private static final AtomicInteger id = new AtomicInteger(0);
public PipeConnectorSubtaskExecutor() {
super(
PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(),
- ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL,
+ ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL.getName() + "-" + id.get(),
+ ThreadName.PIPE_SUBTASK_CALLBACK_EXECUTOR_POOL.getName() + "-" + id.getAndIncrement(),
true);
}
- public PipeConnectorSubtaskExecutor(final int corePoolSize, final ThreadName threadName) {
+ public PipeConnectorSubtaskExecutor(final int corePoolSize, final String threadName) {
super(corePoolSize, threadName, true);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java
index f25d28b0c2c..9eddd27f22e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java
@@ -28,7 +28,7 @@ public class PipeProcessorSubtaskExecutor extends PipeSubtaskExecutor {
public PipeProcessorSubtaskExecutor() {
super(
PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(),
- ThreadName.PIPE_PROCESSOR_EXECUTOR_POOL,
+ ThreadName.PIPE_PROCESSOR_EXECUTOR_POOL.getName(),
false);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
index f0de4d8d58e..e0a71454d23 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
@@ -23,42 +23,44 @@ import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.db.pipe.consensus.PipeConsensusSubtaskExecutor;
import org.apache.iotdb.db.subscription.task.execution.SubscriptionSubtaskExecutor;
+import java.util.function.Supplier;
+
/**
* PipeTaskExecutor is responsible for executing the pipe tasks, and it is scheduled by the
* PipeTaskScheduler. It is a singleton class.
*/
public class PipeSubtaskExecutorManager {
private final PipeProcessorSubtaskExecutor processorExecutor;
- private final PipeConnectorSubtaskExecutor connectorExecutor;
+ private final Supplier<PipeConnectorSubtaskExecutor> connectorExecutorSupplier;
private final SubscriptionSubtaskExecutor subscriptionExecutor;
- private final PipeConsensusSubtaskExecutor consensusExecutor;
+ private final Supplier<PipeConsensusSubtaskExecutor> consensusExecutorSupplier;
public PipeProcessorSubtaskExecutor getProcessorExecutor() {
return processorExecutor;
}
- public PipeConnectorSubtaskExecutor getConnectorExecutor() {
- return connectorExecutor;
+ public Supplier<PipeConnectorSubtaskExecutor> getConnectorExecutorSupplier() {
+ return connectorExecutorSupplier;
}
public SubscriptionSubtaskExecutor getSubscriptionExecutor() {
return subscriptionExecutor;
}
- public PipeConsensusSubtaskExecutor getConsensusExecutor() {
- return consensusExecutor;
+ public Supplier<PipeConsensusSubtaskExecutor> getConsensusExecutorSupplier() {
+ return consensusExecutorSupplier;
}
///////////////////////// Singleton Instance Holder /////////////////////////
private PipeSubtaskExecutorManager() {
processorExecutor = new PipeProcessorSubtaskExecutor();
- connectorExecutor = new PipeConnectorSubtaskExecutor();
+ connectorExecutorSupplier = PipeConnectorSubtaskExecutor::new;
subscriptionExecutor =
SubscriptionConfig.getInstance().getSubscriptionEnabled()
? new SubscriptionSubtaskExecutor()
: null;
- consensusExecutor = new PipeConsensusSubtaskExecutor();
+ consensusExecutorSupplier = PipeConsensusSubtaskExecutor::new;
}
private static class PipeTaskExecutorHolder {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java
index 345e6ab040c..e5ae5dacd27 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java
@@ -28,13 +28,15 @@ import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.exception.PipeException;
+import java.util.function.Supplier;
+
public class PipeTaskConnectorStage extends PipeTaskStage {
protected final String pipeName;
protected final long creationTime;
protected final PipeParameters pipeConnectorParameters;
protected final int regionId;
- protected final PipeConnectorSubtaskExecutor executor;
+ protected final Supplier<? extends PipeConnectorSubtaskExecutor> executor;
protected String connectorSubtaskId;
@@ -43,7 +45,7 @@ public class PipeTaskConnectorStage extends PipeTaskStage {
long creationTime,
PipeParameters pipeConnectorParameters,
int regionId,
- PipeConnectorSubtaskExecutor executor) {
+ Supplier<? extends PipeConnectorSubtaskExecutor> executor) {
this.pipeName = pipeName;
this.creationTime = creationTime;
this.pipeConnectorParameters = pipeConnectorParameters;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
index ecbbc641e4b..6e4a858a370 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.connector;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.pipe.api.event.Event;
import org.slf4j.Logger;
@@ -65,6 +66,8 @@ public class PipeConnectorSubtaskLifeCycle implements AutoCloseable {
if (registeredTaskCount == 0) {
executor.register(subtask);
runningTaskCount = 0;
+
+ PipeDataNodeResourceManager.compaction().registerPipeConnectorSubtaskLifeCycle(this);
}
registeredTaskCount++;
@@ -152,5 +155,7 @@ public class PipeConnectorSubtaskLifeCycle implements AutoCloseable {
@Override
public synchronized void close() {
executor.deregister(subtask.getTaskID());
+
+ PipeDataNodeResourceManager.compaction().deregisterPipeConnectorSubtaskLifeCycle(this);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java
index e556da428ef..e25542135c1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java
@@ -48,6 +48,7 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
public class PipeConnectorSubtaskManager {
@@ -60,7 +61,7 @@ public class PipeConnectorSubtaskManager {
attributeSortedString2SubtaskLifeCycleMap = new HashMap<>();
public synchronized String register(
- final PipeConnectorSubtaskExecutor executor,
+ final Supplier<? extends PipeConnectorSubtaskExecutor> executorSupplier,
final PipeParameters pipeConnectorParameters,
final PipeTaskConnectorRuntimeEnvironment environment) {
final String connectorKey =
@@ -109,6 +110,8 @@ public class PipeConnectorSubtaskManager {
environment.setAttributeSortedString(attributeSortedString);
if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
+ final PipeConnectorSubtaskExecutor executor = executorSupplier.get();
+
final List<PipeConnectorSubtaskLifeCycle> pipeConnectorSubtaskLifeCycleList =
new ArrayList<>(connectorNum);
@@ -169,6 +172,11 @@ public class PipeConnectorSubtaskManager {
pipeConnectorSubtaskLifeCycleList.add(pipeConnectorSubtaskLifeCycle);
}
+ LOGGER.info(
+ "Pipe connector subtasks with attributes {} is bounded with connectorExecutor {} and callbackExecutor {}.",
+ attributeSortedString,
+ executor.getWorkingThreadName(),
+ executor.getCallbackThreadName());
attributeSortedString2SubtaskLifeCycleMap.put(
attributeSortedString, pipeConnectorSubtaskLifeCycleList);
}
@@ -192,10 +200,19 @@ public class PipeConnectorSubtaskManager {
final List<PipeConnectorSubtaskLifeCycle> lifeCycles =
attributeSortedString2SubtaskLifeCycleMap.get(attributeSortedString);
+
+ // Shall not be empty
+ final PipeConnectorSubtaskExecutor executor = lifeCycles.get(0).executor;
+
lifeCycles.removeIf(o -> o.deregister(pipeName, regionId));
if (lifeCycles.isEmpty()) {
attributeSortedString2SubtaskLifeCycleMap.remove(attributeSortedString);
+ executor.shutdown();
+ LOGGER.info(
+ "The executor {} and {} has been successfully shutdown.",
+ executor.getWorkingThreadName(),
+ executor.getCallbackThreadName());
}
PipeEventCommitManager.getInstance().deregister(pipeName, creationTime, regionId);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
index a4f05447eae..409447b5b2f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
@@ -21,23 +21,40 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.connector;
import org.apache.iotdb.commons.pipe.agent.task.connection.BlockingPendingQueue;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
+import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeCompactedTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQueue<Event> {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(PipeRealtimePriorityBlockingQueue.class);
+
private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
private final BlockingDeque<TsFileInsertionEvent> tsfileInsertEventDeque =
@@ -184,6 +201,138 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
return tsfileInsertEventDeque.peek();
}
+ public synchronized void replace(
+ String dataRegionId, Set<TsFileResource> sourceFiles, List<TsFileResource> targetFiles) {
+
+ final int regionId = Integer.parseInt(dataRegionId);
+ final Map<CommitterKey, Set<PipeTsFileInsertionEvent>> eventsToBeRemovedGroupByCommitterKey =
+ tsfileInsertEventDeque.stream()
+ .filter(
+ event ->
+ event instanceof PipeTsFileInsertionEvent
+ && ((PipeTsFileInsertionEvent) event).getRegionId() == regionId)
+ .map(event -> (PipeTsFileInsertionEvent) event)
+ .collect(
+ Collectors.groupingBy(
+ PipeTsFileInsertionEvent::getCommitterKey, Collectors.toSet()))
+ .entrySet()
+ .stream()
+ // Replace if all source files are present in the queue
+ .filter(entry -> entry.getValue().size() == sourceFiles.size())
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ if (eventsToBeRemovedGroupByCommitterKey.isEmpty()) {
+ LOGGER.info(
+ "Region {}: No TsFileInsertionEvents to replace for source files {}",
+ regionId,
+ sourceFiles.stream()
+ .map(TsFileResource::getTsFilePath)
+ .collect(Collectors.joining(", ")));
+ return;
+ }
+
+ final Map<CommitterKey, Set<PipeTsFileInsertionEvent>> eventsToBeAddedGroupByCommitterKey =
+ new HashMap<>();
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
+ eventsToBeRemovedGroupByCommitterKey.entrySet()) {
+ final CommitterKey committerKey = entry.getKey();
+ final PipeTsFileInsertionEvent anyEvent = entry.getValue().stream().findFirst().orElse(null);
+ final Set<PipeTsFileInsertionEvent> newEvents = new HashSet<>();
+ for (int i = 0; i < targetFiles.size(); i++) {
+ newEvents.add(
+ new PipeCompactedTsFileInsertionEvent(
+ committerKey,
+ entry.getValue(),
+ anyEvent,
+ targetFiles.get(i),
+ i == targetFiles.size() - 1));
+ }
+ eventsToBeAddedGroupByCommitterKey.put(committerKey, newEvents);
+ }
+
+ // Handling new events
+ final Set<PipeTsFileInsertionEvent> successfullyReferenceIncreasedEvents = new HashSet<>();
+ final AtomicBoolean
+ allSuccess = // To track if all events successfully increased the reference count
+ new AtomicBoolean(true);
+ outerLoop:
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> committerKeySetEntry :
+ eventsToBeAddedGroupByCommitterKey.entrySet()) {
+ for (final PipeTsFileInsertionEvent event : committerKeySetEntry.getValue()) {
+ if (event != null) {
+ try {
+ if (!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName())) {
+ allSuccess.set(false);
+ break outerLoop;
+ } else {
+ successfullyReferenceIncreasedEvents.add(event);
+ }
+ } catch (final Exception e) {
+ allSuccess.set(false);
+ break outerLoop;
+ }
+ }
+ }
+ }
+ if (!allSuccess.get()) {
+ // If any event failed to increase the reference count,
+ // we need to decrease the reference count for all successfully increased events
+ for (final PipeTsFileInsertionEvent event : successfullyReferenceIncreasedEvents) {
+ try {
+ event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to decrease reference count for event {} in PipeRealtimePriorityBlockingQueue",
+ event,
+ e);
+ }
+ }
+ return; // Exit early if any event failed to increase the reference count
+ } else {
+ // If all events successfully increased reference count,
+ // we can proceed to add them to the deque
+ for (final PipeTsFileInsertionEvent event : successfullyReferenceIncreasedEvents) {
+ tsfileInsertEventDeque.add(event);
+ eventCounter.increaseEventCount(event);
+ }
+ }
+
+ // Handling old events
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
+ eventsToBeRemovedGroupByCommitterKey.entrySet()) {
+ for (final PipeTsFileInsertionEvent event : entry.getValue()) {
+ if (event != null) {
+ try {
+ event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to decrease reference count for event {} in PipeRealtimePriorityBlockingQueue",
+ event,
+ e);
+ }
+ eventCounter.decreaseEventCount(event);
+ }
+ }
+ }
+ final Set<PipeTsFileInsertionEvent> eventsToRemove = new HashSet<>();
+ for (Set<PipeTsFileInsertionEvent> pipeTsFileInsertionEvents :
+ eventsToBeRemovedGroupByCommitterKey.values()) {
+ eventsToRemove.addAll(pipeTsFileInsertionEvents);
+ }
+ tsfileInsertEventDeque.removeIf(eventsToRemove::contains);
+
+ LOGGER.info(
+ "Region {}: Replaced TsFileInsertionEvents {} with {}",
+ regionId,
+ eventsToBeRemovedGroupByCommitterKey.values().stream()
+ .flatMap(Set::stream)
+ .map(PipeTsFileInsertionEvent::coreReportMessage)
+ .collect(Collectors.joining(", ")),
+ eventsToBeAddedGroupByCommitterKey.values().stream()
+ .flatMap(Set::stream)
+ .map(PipeTsFileInsertionEvent::coreReportMessage)
+ .collect(Collectors.joining(", ")));
+ }
+
@Override
public void clear() {
super.clear();
@@ -244,7 +393,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
return tsfileInsertEventDeque.size();
}
- public void setOfferTsFileCounter(AtomicInteger offerTsFileCounter) {
+ public synchronized void setOfferTsFileCounter(AtomicInteger offerTsFileCounter) {
this.offerTsFileCounter = offerTsFileCounter;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java
index 9cb1b0a5cd0..2fc41a61866 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.commons.client.ClientPoolFactory;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient;
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.connector.client.IoTDBClientManager;
@@ -48,7 +50,9 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY;
@@ -70,7 +74,12 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
// receiverAttributes -> IClientManager<TEndPoint, AsyncPipeDataTransferServiceClient>
private static final Map<String, IClientManager<TEndPoint, AsyncPipeDataTransferServiceClient>>
ASYNC_PIPE_DATA_TRANSFER_CLIENT_MANAGER_HOLDER = new ConcurrentHashMap<>();
+ private static final Map<String, ExecutorService> TS_FILE_ASYNC_EXECUTOR_HOLDER =
+ new ConcurrentHashMap<>();
+ private static final AtomicInteger id = new AtomicInteger(0);
+
private final IClientManager<TEndPoint, AsyncPipeDataTransferServiceClient> endPoint2Client;
+ private ExecutorService executor;
private final LoadBalancer loadBalancer;
@@ -123,6 +132,17 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
}
endPoint2Client = ASYNC_PIPE_DATA_TRANSFER_CLIENT_MANAGER_HOLDER.get(receiverAttributes);
+ if (isTSFileUsed) {
+ if (!TS_FILE_ASYNC_EXECUTOR_HOLDER.containsKey(receiverAttributes)) {
+ TS_FILE_ASYNC_EXECUTOR_HOLDER.putIfAbsent(
+ receiverAttributes,
+ IoTDBThreadPoolFactory.newFixedThreadPool(
+ PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize(),
+ ThreadName.PIPE_TSFILE_ASYNC_SEND_POOL.getName() + "-" + id.getAndIncrement()));
+ }
+ executor = TS_FILE_ASYNC_EXECUTOR_HOLDER.get(receiverAttributes);
+ }
+
RECEIVER_ATTRIBUTES_REF_COUNT.compute(
receiverAttributes, (attributes, refCount) -> refCount == null ? 1 : refCount + 1);
}
@@ -336,6 +356,10 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
LEADER_CACHE_MANAGER.updateLeaderEndPoint(deviceId, endPoint);
}
+ public ExecutorService getExecutor() {
+ return executor;
+ }
+
public void close() {
isClosed = true;
synchronized (IoTDBDataNodeAsyncClientManager.class) {
@@ -352,6 +376,18 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
LOGGER.warn("Failed to close client manager.", e);
}
}
+
+ final ExecutorService executor =
+ TS_FILE_ASYNC_EXECUTOR_HOLDER.remove(receiverAttributes);
+ if (executor != null) {
+ try {
+ executor.shutdown();
+ LOGGER.info("Successfully shutdown executor {}.", executor);
+ } catch (final Exception e) {
+ LOGGER.warn("Failed to shutdown executor {}.", executor);
+ }
+ }
+
return null;
}
return refCount - 1;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
index 1cb9f50d92d..8d52055fcc9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.pipe.connector.payload.evolvable.batch;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.memory.PipeDynamicMemoryBlock;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlockType;
import org.apache.iotdb.db.pipe.resource.memory.PipeModelFixedMemoryBlock;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
@@ -47,7 +48,7 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
private long firstEventProcessingTime = Long.MIN_VALUE;
protected long totalBufferSize = 0;
- private final PipeModelFixedMemoryBlock allocatedMemoryBlock;
+ private final PipeDynamicMemoryBlock allocatedMemoryBlock;
protected volatile boolean isClosed = false;
@@ -60,11 +61,9 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
// limit in buffer size
this.allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(
- requestMaxBatchSizeInBytes, PipeMemoryBlockType.BATCH);
+ pipeModelFixedMemoryBlock.registerPipeBatchMemoryBlock(requestMaxBatchSizeInBytes);
- if (getMaxBatchSizeInBytes() != requestMaxBatchSizeInBytes) {
+ if (getMaxBatchSizeInBytes() != allocatedMemoryBlock.getMemoryUsageInBytes()) {
LOGGER.info(
"PipeTabletEventBatch: the max batch size is adjusted from {} to {} due to the "
+ "memory restriction",
@@ -127,10 +126,17 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
throws WALPipeException, IOException;
public boolean shouldEmit() {
- return totalBufferSize >= getMaxBatchSizeInBytes()
- || System.currentTimeMillis() - firstEventProcessingTime >= maxDelayInMs;
+ final long diff = System.currentTimeMillis() - firstEventProcessingTime;
+ if (totalBufferSize >= getMaxBatchSizeInBytes() || diff >= maxDelayInMs) {
+ allocatedMemoryBlock.updateCurrentMemoryEfficiencyAdjustMem((double) diff / maxDelayInMs);
+ recordMetric(diff, totalBufferSize);
+ return true;
+ }
+ return false;
}
+ protected abstract void recordMetric(final long timeInterval, final long bufferSize);
+
private long getMaxBatchSizeInBytes() {
return allocatedMemoryBlock.getMemoryUsageInBytes();
}
@@ -194,9 +200,22 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
}
try {
- pipeModelFixedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(0L, PipeMemoryBlockType.BATCH);
+ long batchSize = PipeDataNodeResourceManager.memory().getAllocatedMemorySizeInBytesOfBatch();
+ for (long i = batchSize; i > 0; i = i / 2) {
+ try {
+ pipeModelFixedMemoryBlock =
+ PipeDataNodeResourceManager.memory()
+ .forceAllocateForModelFixedMemoryBlock(i, PipeMemoryBlockType.BATCH);
+
+ LOGGER.info("pipe model fixed memory block initialized with size: {} bytes", i);
+ return;
+ } catch (Exception ignore) {
+ // ignore the exception and try to allocate a smaller size
+ LOGGER.info(
+ "pipe model fixed memory block initialized with size: {} bytes failed, try smaller size",
+ i);
+ }
+ }
} catch (Exception e) {
LOGGER.error("init pipe model fixed memory block failed", e);
// If the allocation fails, we still need to create a default memory block to avoid NPE.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
index 292fc018ed7..70ffaddb180 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletBatchReq;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
@@ -70,6 +71,12 @@ public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
return true;
}
+ @Override
+ protected void recordMetric(long timeInterval, long bufferSize) {
+ PipeDataRegionConnectorMetrics.tabletBatchTimeIntervalHistogram.update(timeInterval);
+ PipeDataRegionConnectorMetrics.tabletBatchSizeHistogram.update(bufferSize);
+ }
+
@Override
public synchronized void onSuccess() {
super.onSuccess();
@@ -102,8 +109,7 @@ public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
(PipeInsertNodeTabletInsertionEvent) event;
// Read the bytebuffer from the wal file and transfer it directly without serializing or
// deserializing if possible
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
if (Objects.isNull(insertNode)) {
buffer = pipeInsertNodeTabletInsertionEvent.getByteBuffer();
binaryBuffers.add(buffer);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
index a03f2bd970a..405245c6acb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.pipe.connector.util.PipeTabletEventSorter;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
@@ -180,6 +181,12 @@ public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch {
return true;
}
+ @Override
+ protected void recordMetric(long timeInterval, long bufferSize) {
+ PipeDataRegionConnectorMetrics.tsFileBatchTimeIntervalHistogram.update(timeInterval);
+ PipeDataRegionConnectorMetrics.tsFileBatchSizeHistogram.update(bufferSize);
+ }
+
private void bufferTablet(
final String pipeName,
final long creationTime,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
index 580dbe2ed4c..553a04cbfcd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
@@ -66,7 +66,6 @@ public class PipeTransferPlanNodeReq extends TPipeTransferReq {
planNodeReq.version = transferReq.version;
planNodeReq.type = transferReq.type;
- planNodeReq.body = transferReq.body;
return planNodeReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
index 8090f650489..27664cdbac7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
@@ -182,7 +182,6 @@ public class PipeTransferTabletBatchReq extends TPipeTransferReq {
batchReq.version = transferReq.version;
batchReq.type = transferReq.type;
- batchReq.body = transferReq.body;
return batchReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
index 5e9e0a39103..4deac8a5cf6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
@@ -89,7 +89,6 @@ public class PipeTransferTabletBinaryReq extends TPipeTransferReq {
binaryReq.version = transferReq.version;
binaryReq.type = transferReq.type;
- binaryReq.body = transferReq.body;
return binaryReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
index c45417ba99d..376ca121c80 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
@@ -96,7 +96,6 @@ public class PipeTransferTabletInsertNodeReq extends TPipeTransferReq {
insertNodeReq.version = transferReq.version;
insertNodeReq.type = transferReq.type;
- insertNodeReq.body = transferReq.body;
return insertNodeReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
index 61790f883ae..f1e32785df7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
@@ -132,7 +132,6 @@ public class PipeTransferTabletRawReq extends TPipeTransferReq {
tabletReq.version = transferReq.version;
tabletReq.type = transferReq.type;
- tabletReq.body = transferReq.body;
return tabletReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
index ee35ab46306..bc4805e8c8e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
@@ -168,8 +168,7 @@ public class IoTDBDataRegionAirGapConnector extends IoTDBDataNodeAirGapConnector
final AirGapSocket socket,
final PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent)
throws PipeException, WALPipeException, IOException {
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final byte[] bytes =
Objects.isNull(insertNode)
? PipeTransferTabletBinaryReq.toTPipeTransferBytes(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
index a74057c75c7..1756da36db7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
@@ -277,8 +277,7 @@ public class PipeConsensusAsyncConnector extends IoTDBConnector implements Conse
return;
}
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final ProgressIndex progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
final TPipeConsensusTransferReq pipeConsensusTransferReq =
Objects.isNull(insertNode)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
index f4690783fe1..6186d9f671a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
@@ -252,7 +252,7 @@ public class PipeConsensusSyncConnector extends IoTDBConnector {
try (final SyncPipeConsensusServiceClient syncPipeConsensusServiceClient =
syncRetryClientManager.borrowClient(getFollowerUrl())) {
- insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
if (insertNode != null) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
index 870cea0ee87..a3f8fe557b7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
@@ -196,8 +196,7 @@ public abstract class PipeConsensusTransferBatchReqBuilder implements AutoClosea
// Read the bytebuffer from the wal file and transfer it directly without serializing or
// deserializing if possible
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
// PipeConsensus will transfer binary data to TPipeConsensusTransferReq
final ProgressIndex progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
if (Objects.isNull(insertNode)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
index ac639fd2257..0dacf0cb912 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
@@ -74,8 +74,6 @@ import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -97,11 +95,10 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
"Exception occurred while sending to receiver %s:%s.";
private static final boolean isSplitTSFileBatchModeEnabled = true;
- private static final ExecutorService executor =
- Executors.newFixedThreadPool(PipeConfig.getInstance().getPipeAsyncConnectorMaxClientNumber());
private final IoTDBDataRegionSyncConnector syncConnector = new IoTDBDataRegionSyncConnector();
private final BlockingQueue<Event> retryEventQueue = new LinkedBlockingQueue<>();
+ private final BlockingQueue<TsFileInsertionEvent> retryTsFileQueue = new LinkedBlockingQueue<>();
private final PipeDataRegionEventCounter retryEventQueueEventCounter =
new PipeDataRegionEventCounter();
@@ -265,8 +262,7 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
return false;
}
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final TPipeTransferReq pipeTransferReq =
compressIfNeeded(
Objects.isNull(insertNode)
@@ -418,9 +414,9 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
}
return null;
},
- executor);
+ transferTsFileClientManager.getExecutor());
- if (PipeConfig.getInstance().isTransferTsFileSync()) {
+ if (PipeConfig.getInstance().isTransferTsFileSync() || !isRealtimeFirst) {
try {
completableFuture.get();
} catch (InterruptedException e) {
@@ -498,7 +494,7 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
* @see PipeConnector#transfer(TsFileInsertionEvent) for more details.
*/
private void transferQueuedEventsIfNecessary(final boolean forced) {
- if (retryEventQueue.isEmpty()
+ if ((retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty())
|| (!forced
&& retryEventQueueEventCounter.getTabletInsertionEventCount()
< PipeConfig.getInstance()
@@ -506,38 +502,49 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
&& retryEventQueueEventCounter.getTsFileInsertionEventCount()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold()
- && retryEventQueue.size()
+ && retryEventQueue.size() + retryTsFileQueue.size()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold())) {
return;
}
final long retryStartTime = System.currentTimeMillis();
- final int remainingEvents = retryEventQueue.size();
- while (!retryEventQueue.isEmpty()) {
+ final int remainingEvents = retryEventQueue.size() + retryTsFileQueue.size();
+ while (!retryEventQueue.isEmpty() || !retryTsFileQueue.isEmpty()) {
synchronized (this) {
if (isClosed.get()) {
return;
}
- if (retryEventQueue.isEmpty()) {
+ if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
break;
}
- final Event peekedEvent = retryEventQueue.peek();
+ final Event peekedEvent;
+ final Event polledEvent;
+ if (!retryEventQueue.isEmpty()) {
+ peekedEvent = retryEventQueue.peek();
+
+ if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) {
+ retryTransfer((PipeInsertNodeTabletInsertionEvent) peekedEvent);
+ } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) {
+ retryTransfer((PipeRawTabletInsertionEvent) peekedEvent);
+ } else {
+ LOGGER.warn(
+ "IoTDBThriftAsyncConnector does not support transfer generic event: {}.",
+ peekedEvent);
+ }
- if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) {
- retryTransfer((PipeInsertNodeTabletInsertionEvent) peekedEvent);
- } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) {
- retryTransfer((PipeRawTabletInsertionEvent) peekedEvent);
- } else if (peekedEvent instanceof PipeTsFileInsertionEvent) {
- retryTransfer((PipeTsFileInsertionEvent) peekedEvent);
+ polledEvent = retryEventQueue.poll();
} else {
- LOGGER.warn(
- "IoTDBThriftAsyncConnector does not support transfer generic event: {}.",
- peekedEvent);
+ if (transferTsFileCounter.get()
+ >= PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize()) {
+ return;
+ }
+ peekedEvent = retryTsFileQueue.peek();
+ retryTransfer((PipeTsFileInsertionEvent) peekedEvent);
+ polledEvent = retryTsFileQueue.poll();
}
- final Event polledEvent = retryEventQueue.poll();
retryEventQueueEventCounter.decreaseEventCount(polledEvent);
if (polledEvent != peekedEvent) {
LOGGER.error(
@@ -560,16 +567,16 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
&& retryEventQueueEventCounter.getTsFileInsertionEventCount()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold()
- && retryEventQueue.size()
+ && retryEventQueue.size() + retryTsFileQueue.size()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold()) {
return;
}
- if (remainingEvents <= retryEventQueue.size()) {
+ if (remainingEvents <= retryEventQueue.size() + retryTsFileQueue.size()) {
throw new PipeException(
"Failed to retry transferring events in the retry queue. Remaining events: "
- + retryEventQueue.size()
+ + (retryEventQueue.size() + retryTsFileQueue.size())
+ " (tablet events: "
+ retryEventQueueEventCounter.getTabletInsertionEventCount()
+ ", tsfile events: "
@@ -647,8 +654,14 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
return;
}
- retryEventQueue.offer(event);
- retryEventQueueEventCounter.increaseEventCount(event);
+ if (event instanceof PipeTsFileInsertionEvent) {
+ retryTsFileQueue.offer((PipeTsFileInsertionEvent) event);
+ retryEventQueueEventCounter.increaseEventCount(event);
+ } else {
+ retryEventQueue.offer(event);
+ retryEventQueueEventCounter.increaseEventCount(event);
+ }
+
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Added event {} to retry queue.", event);
}
@@ -688,6 +701,19 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
}
return false;
});
+
+ retryTsFileQueue.removeIf(
+ event -> {
+ if (event instanceof EnrichedEvent
+ && pipeNameToDrop.equals(((EnrichedEvent) event).getPipeName())
+ && regionId == ((EnrichedEvent) event).getRegionId()) {
+ ((EnrichedEvent) event)
+ .clearReferenceCount(IoTDBDataRegionAsyncConnector.class.getName());
+ retryEventQueueEventCounter.decreaseEventCount(event);
+ return true;
+ }
+ return false;
+ });
}
@Override
@@ -730,8 +756,9 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
}
public synchronized void clearRetryEventsReferenceCount() {
- while (!retryEventQueue.isEmpty()) {
- final Event event = retryEventQueue.poll();
+ while (!retryEventQueue.isEmpty() || !retryTsFileQueue.isEmpty()) {
+ final Event event =
+ retryTsFileQueue.isEmpty() ? retryEventQueue.poll() : retryTsFileQueue.poll();
retryEventQueueEventCounter.decreaseEventCount(event);
if (event instanceof EnrichedEvent) {
((EnrichedEvent) event).clearReferenceCount(IoTDBDataRegionAsyncConnector.class.getName());
@@ -742,7 +769,7 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
//////////////////////// APIs provided for metric framework ////////////////////////
public int getRetryEventQueueSize() {
- return retryEventQueue.size();
+ return retryEventQueue.size() + retryTsFileQueue.size();
}
public int getBatchSize() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java
index f77d41b6662..b3eb223e58e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java
@@ -104,7 +104,7 @@ public abstract class PipeTransferTabletInsertionEventHandler extends PipeTransf
? ((EnrichedEvent) event).coreReportMessage()
: event.toString(),
event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitterKey() : null,
- event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitId() : null,
+ event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitIds() : null,
exception);
} finally {
connector.addFailureEventToRetryQueue(event);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
index ccce6ba0aa2..7353ea91e91 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
@@ -47,7 +47,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Arrays;
@@ -78,8 +77,8 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
private final boolean transferMod;
private final int readFileBufferSize;
- private final PipeTsFileMemoryBlock memoryBlock;
- private final byte[] readBuffer;
+ private PipeTsFileMemoryBlock memoryBlock;
+ private byte[] readBuffer;
private long position;
private RandomAccessFile reader;
@@ -98,7 +97,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
final File tsFile,
final File modFile,
final boolean transferMod)
- throws FileNotFoundException, InterruptedException {
+ throws InterruptedException {
super(connector);
this.pipeName2WeightMap = pipeName2WeightMap;
@@ -124,20 +123,8 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
Math.min(
PipeConfig.getInstance().getPipeConnectorReadFileBufferSize(),
transferMod ? Math.max(tsFile.length(), modFile.length()) : tsFile.length());
- memoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForTsFileWithRetry(
- PipeConfig.getInstance().isPipeConnectorReadFileBufferMemoryControlEnabled()
- ? readFileBufferSize
- : 0);
- readBuffer = new byte[readFileBufferSize];
position = 0;
- reader =
- Objects.nonNull(modFile)
- ? new RandomAccessFile(modFile, "r")
- : new RandomAccessFile(tsFile, "r");
-
isSealSignalSent = new AtomicBoolean(false);
}
@@ -145,6 +132,24 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
final IoTDBDataNodeAsyncClientManager clientManager,
final AsyncPipeDataTransferServiceClient client)
throws TException, IOException {
+ // Delay creation of resources to avoid OOM or too many open files
+ if (readBuffer == null) {
+ memoryBlock =
+ PipeDataNodeResourceManager.memory()
+ .forceAllocateForTsFileWithRetry(
+ PipeConfig.getInstance().isPipeConnectorReadFileBufferMemoryControlEnabled()
+ ? readFileBufferSize
+ : 0);
+ readBuffer = new byte[readFileBufferSize];
+ }
+
+ if (reader == null) {
+ reader =
+ Objects.nonNull(modFile)
+ ? new RandomAccessFile(modFile, "r")
+ : new RandomAccessFile(tsFile, "r");
+ }
+
this.clientManager = clientManager;
this.client = client;
@@ -285,7 +290,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
"Successfully transferred file {} (committer key={}, commit id={}, reference count={}).",
tsFile,
events.stream().map(EnrichedEvent::getCommitterKey).collect(Collectors.toList()),
- events.stream().map(EnrichedEvent::getCommitId).collect(Collectors.toList()),
+ events.stream().map(EnrichedEvent::getCommitIds).collect(Collectors.toList()),
referenceCount);
} else {
LOGGER.info(
@@ -350,7 +355,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
"Failed to transfer TsFileInsertionEvent {} (committer key {}, commit id {}).",
tsFile,
events.stream().map(EnrichedEvent::getCommitterKey).collect(Collectors.toList()),
- events.stream().map(EnrichedEvent::getCommitId).collect(Collectors.toList()),
+ events.stream().map(EnrichedEvent::getCommitIds).collect(Collectors.toList()),
exception);
} else {
LOGGER.warn(
@@ -427,7 +432,9 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
@Override
public void close() {
super.close();
- memoryBlock.close();
+ if (memoryBlock != null) {
+ memoryBlock.close();
+ }
}
/**
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
index 784ee14a55a..d996cac2820 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
@@ -294,8 +294,7 @@ public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
// getDeviceId() may return null for InsertRowsNode, will be equal to getClient(null)
clientAndStatus = clientManager.getClient(pipeInsertNodeTabletInsertionEvent.getDeviceId());
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final TPipeTransferReq req =
compressIfNeeded(
insertNode != null
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
index 7d28f8609ad..365950c5109 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
@@ -129,8 +129,7 @@ public class WriteBackConnector implements PipeConnector {
throws PipeException, WALPipeException {
final TSStatus status;
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
if (Objects.isNull(insertNode)) {
status =
PipeDataNodeAgent.receiver()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java
index 72782ad228e..d50e08463e2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java
@@ -22,11 +22,15 @@ import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
+import java.util.concurrent.atomic.AtomicInteger;
+
public class PipeConsensusSubtaskExecutor extends PipeConnectorSubtaskExecutor {
+ private static final AtomicInteger id = new AtomicInteger(0);
+
public PipeConsensusSubtaskExecutor() {
super(
PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(),
- ThreadName.PIPE_CONSENSUS_EXECUTOR_POOL);
+ ThreadName.PIPE_CONSENSUS_EXECUTOR_POOL + "-" + id.getAndIncrement());
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
index 80ac86310b0..ed7d59f0b35 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.ReferenceTrackableEvent;
import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
@@ -37,8 +38,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryPosition;
import org.apache.iotdb.pipe.api.access.Row;
import org.apache.iotdb.pipe.api.collector.RowCollector;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
@@ -69,52 +68,26 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
LoggerFactory.getLogger(PipeInsertNodeTabletInsertionEvent.class);
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(PipeInsertNodeTabletInsertionEvent.class)
- + RamUsageEstimator.shallowSizeOfInstance(WALEntryHandler.class)
- + RamUsageEstimator.shallowSizeOfInstance(WALEntryPosition.class)
+ RamUsageEstimator.shallowSizeOfInstance(AtomicInteger.class)
+ RamUsageEstimator.shallowSizeOfInstance(AtomicBoolean.class);
- private final WALEntryHandler walEntryHandler;
- private final boolean isAligned;
- private final boolean isGeneratedByPipe;
-
private final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock;
private volatile List<Tablet> tablets;
private List<TabletInsertionDataContainer> dataContainers;
- private final PartialPath devicePath;
+ private InsertNode insertNode;
private ProgressIndex progressIndex;
private long extractTime = 0;
- public PipeInsertNodeTabletInsertionEvent(
- final WALEntryHandler walEntryHandler,
- final PartialPath devicePath,
- final ProgressIndex progressIndex,
- final boolean isAligned,
- final boolean isGeneratedByPipe) {
- this(
- walEntryHandler,
- devicePath,
- progressIndex,
- isAligned,
- isGeneratedByPipe,
- null,
- 0,
- null,
- null,
- Long.MIN_VALUE,
- Long.MAX_VALUE);
+ public PipeInsertNodeTabletInsertionEvent(final InsertNode insertNode) {
+ this(insertNode, null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE);
}
private PipeInsertNodeTabletInsertionEvent(
- final WALEntryHandler walEntryHandler,
- final PartialPath devicePath,
- final ProgressIndex progressIndex,
- final boolean isAligned,
- final boolean isGeneratedByPipe,
+ final InsertNode insertNode,
final String pipeName,
final long creationTime,
final PipeTaskMeta pipeTaskMeta,
@@ -122,34 +95,24 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
final long startTime,
final long endTime) {
super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime);
- this.walEntryHandler = walEntryHandler;
// Record device path here so there's no need to get it from InsertNode cache later.
- this.devicePath = devicePath;
- this.progressIndex = progressIndex;
- this.isAligned = isAligned;
- this.isGeneratedByPipe = isGeneratedByPipe;
-
+ this.progressIndex = insertNode.getProgressIndex();
+ this.insertNode = insertNode;
this.allocatedMemoryBlock = new AtomicReference<>();
}
- public InsertNode getInsertNode() throws WALPipeException {
- return walEntryHandler.getInsertNode();
+ public InsertNode getInsertNode() {
+ return insertNode;
}
public ByteBuffer getByteBuffer() throws WALPipeException {
- return walEntryHandler.getByteBuffer();
- }
-
- // This method is a pre-determination of whether to use binary transfers.
- // If the insert node is null in cache, it means that we need to read the bytebuffer from the wal,
- // and when the pattern is default, we can transfer the bytebuffer directly without serializing or
- // deserializing
- public InsertNode getInsertNodeViaCacheIfPossible() {
- return walEntryHandler.getInsertNodeViaCacheIfPossible();
+ return insertNode.serializeToByteBuffer();
}
public String getDeviceId() {
- return Objects.nonNull(devicePath) ? devicePath.getFullPath() : null;
+ return Objects.nonNull(insertNode.getDevicePath())
+ ? insertNode.getDevicePath().getFullPath()
+ : null;
}
/////////////////////////// EnrichedEvent ///////////////////////////
@@ -158,19 +121,16 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
extractTime = System.nanoTime();
try {
- PipeDataNodeResourceManager.wal().pin(walEntryHandler);
if (Objects.nonNull(pipeName)) {
PipeDataNodeSinglePipeMetrics.getInstance()
.increaseInsertNodeEventCount(pipeName, creationTime);
- PipeDataNodeAgent.task().addFloatingMemoryUsageInByte(pipeName, ramBytesUsed());
+ PipeDataNodeAgent.task()
+ .addFloatingMemoryUsageInByte(pipeName, creationTime, ramBytesUsed());
}
return true;
} catch (final Exception e) {
LOGGER.warn(
- String.format(
- "Increase reference count for memTable %d error. Holder Message: %s",
- walEntryHandler.getMemTableId(), holderMessage),
- e);
+ String.format("Increase reference count error. Holder Message: %s", holderMessage), e);
return false;
}
}
@@ -178,7 +138,6 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
try {
- PipeDataNodeResourceManager.wal().unpin(walEntryHandler);
// release the containers' memory and close memory block
if (dataContainers != null) {
dataContainers.clear();
@@ -188,17 +147,16 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
return true;
} catch (final Exception e) {
LOGGER.warn(
- String.format(
- "Decrease reference count for memtable %d error. Holder Message: %s",
- walEntryHandler.getMemTableId(), holderMessage),
- e);
+ String.format("Decrease reference count error. Holder Message: %s", holderMessage), e);
return false;
} finally {
if (Objects.nonNull(pipeName)) {
- PipeDataNodeAgent.task().decreaseFloatingMemoryUsageInByte(pipeName, ramBytesUsed());
+ PipeDataNodeAgent.task()
+ .decreaseFloatingMemoryUsageInByte(pipeName, creationTime, ramBytesUsed());
PipeDataNodeSinglePipeMetrics.getInstance()
.decreaseInsertNodeEventCount(pipeName, creationTime, System.nanoTime() - extractTime);
}
+ insertNode = null;
}
}
@@ -221,28 +179,18 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
final long startTime,
final long endTime) {
return new PipeInsertNodeTabletInsertionEvent(
- walEntryHandler,
- devicePath,
- progressIndex,
- isAligned,
- isGeneratedByPipe,
- pipeName,
- creationTime,
- pipeTaskMeta,
- pattern,
- startTime,
- endTime);
+ insertNode, pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime);
}
@Override
public boolean isGeneratedByPipe() {
- return isGeneratedByPipe;
+ return insertNode.isGeneratedByPipe();
}
@Override
public boolean mayEventTimeOverlappedWithTimeRange() {
try {
- final InsertNode insertNode = getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = getInsertNode();
if (Objects.isNull(insertNode)) {
return true;
}
@@ -286,7 +234,7 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public boolean mayEventPathsOverlappedWithPattern() {
try {
- final InsertNode insertNode = getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = getInsertNode();
if (Objects.isNull(insertNode)) {
return true;
}
@@ -439,8 +387,11 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public String toString() {
return String.format(
- "PipeInsertNodeTabletInsertionEvent{walEntryHandler=%s, progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s, dataContainers=%s}",
- walEntryHandler, progressIndex, isAligned, isGeneratedByPipe, dataContainers)
+ "PipeInsertNodeTabletInsertionEvent{progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s, dataContainers=%s}",
+ progressIndex,
+ Objects.nonNull(insertNode) ? insertNode.isAligned() : null,
+ Objects.nonNull(insertNode) ? insertNode.isGeneratedByPipe() : null,
+ dataContainers)
+ " - "
+ super.toString();
}
@@ -448,8 +399,10 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public String coreReportMessage() {
return String.format(
- "PipeInsertNodeTabletInsertionEvent{walEntryHandler=%s, progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s}",
- walEntryHandler, progressIndex, isAligned, isGeneratedByPipe)
+ "PipeInsertNodeTabletInsertionEvent{progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s}",
+ progressIndex,
+ Objects.nonNull(insertNode) ? insertNode.isAligned() : null,
+ Objects.nonNull(insertNode) ? insertNode.isGeneratedByPipe() : null)
+ " - "
+ super.coreReportMessage();
}
@@ -462,7 +415,7 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public long ramBytesUsed() {
return INSTANCE_SIZE
- + (Objects.nonNull(devicePath) ? PartialPath.estimateSize(devicePath) : 0)
+ + (Objects.nonNull(insertNode) ? InsertNodeMemoryEstimator.sizeOf(insertNode) : 0)
+ (Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0);
}
@@ -476,28 +429,24 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public PipeEventResource eventResourceBuilder() {
return new PipeInsertNodeTabletInsertionEventResource(
- this.isReleased, this.referenceCount, this.walEntryHandler, this.allocatedMemoryBlock);
+ this.isReleased, this.referenceCount, this.allocatedMemoryBlock);
}
private static class PipeInsertNodeTabletInsertionEventResource extends PipeEventResource {
- private final WALEntryHandler walEntryHandler;
private final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock;
private PipeInsertNodeTabletInsertionEventResource(
final AtomicBoolean isReleased,
final AtomicInteger referenceCount,
- final WALEntryHandler walEntryHandler,
final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock) {
super(isReleased, referenceCount);
- this.walEntryHandler = walEntryHandler;
this.allocatedMemoryBlock = allocatedMemoryBlock;
}
@Override
protected void finalizeResource() {
try {
- PipeDataNodeResourceManager.wal().unpin(walEntryHandler);
allocatedMemoryBlock.getAndUpdate(
memoryBlock -> {
if (Objects.nonNull(memoryBlock)) {
@@ -506,8 +455,7 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
return null;
});
} catch (final Exception e) {
- LOGGER.warn(
- "Decrease reference count for memTable {} error.", walEntryHandler.getMemTableId(), e);
+ LOGGER.warn("Decrease reference count error.", e);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
index ca78dd7d50a..91d38cf3361 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
@@ -19,23 +19,16 @@
package org.apache.iotdb.db.pipe.event.common.terminate;
-import org.apache.iotdb.common.rpc.thrift.TFlushReq;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTask;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
-import org.apache.iotdb.db.storageengine.StorageEngine;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicLong;
/**
* The {@link PipeTerminateEvent} is an {@link EnrichedEvent} that controls the termination of pipe,
@@ -45,40 +38,6 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class PipeTerminateEvent extends EnrichedEvent {
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeTerminateEvent.class);
-
- private static final AtomicLong PROGRESS_REPORT_COUNT = new AtomicLong(0);
- private static final AtomicLong LAST_PROGRESS_REPORT_TIME = new AtomicLong(0);
-
- public static void flushDataRegionIfNeeded() {
- if (PROGRESS_REPORT_COUNT.get() > 0
- && PROGRESS_REPORT_COUNT.get()
- > PipeConfig.getInstance().getPipeFlushAfterTerminateCount()) {
- flushDataRegion();
- return;
- }
-
- if (LAST_PROGRESS_REPORT_TIME.get() > 0
- && System.currentTimeMillis() - LAST_PROGRESS_REPORT_TIME.get()
- > PipeConfig.getInstance().getPipeFlushAfterLastTerminateSeconds() * 1000L) {
- flushDataRegion();
- }
- }
-
- private static void flushDataRegion() {
- try {
- StorageEngine.getInstance().operateFlush(new TFlushReq());
- PROGRESS_REPORT_COUNT.set(0);
- LAST_PROGRESS_REPORT_TIME.set(0);
- LOGGER.info("Force flush all data regions because of last progress report time.");
- } catch (final Exception e) {
- LOGGER.warn(
- "Failed to flush all data regions, please check the error message: {}",
- e.getMessage(),
- e);
- }
- }
-
private final int dataRegionId;
public PipeTerminateEvent(
@@ -135,9 +94,6 @@ public class PipeTerminateEvent extends EnrichedEvent {
@Override
public void reportProgress() {
- PROGRESS_REPORT_COUNT.incrementAndGet();
- LAST_PROGRESS_REPORT_TIME.set(System.currentTimeMillis());
-
// To avoid deadlock
CompletableFuture.runAsync(
() -> PipeDataNodeAgent.task().markCompleted(pipeName, dataRegionId));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
new file mode 100644
index 00000000000..9160ddf55af
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.event.common.tsfile;
+
+import org.apache.iotdb.commons.consensus.index.ProgressIndex;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
+import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import java.io.File;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class PipeCompactedTsFileInsertionEvent extends PipeTsFileInsertionEvent {
+
+ private final String dataRegionId;
+ private final Set<String> originFilePaths;
+ private final List<Long> commitIds;
+
+ public PipeCompactedTsFileInsertionEvent(
+ final CommitterKey committerKey,
+ final Set<PipeTsFileInsertionEvent> originalEvents,
+ final PipeTsFileInsertionEvent anyOfOriginalEvents,
+ final TsFileResource tsFileResource,
+ final boolean shouldReportProgress) {
+ super(
+ tsFileResource,
+ null,
+ bindIsWithMod(originalEvents),
+ bindIsLoaded(originalEvents),
+ bindIsGeneratedByHistoricalExtractor(originalEvents),
+ committerKey.getPipeName(),
+ committerKey.getCreationTime(),
+ anyOfOriginalEvents.getPipeTaskMeta(),
+ anyOfOriginalEvents.getPipePattern(),
+ anyOfOriginalEvents.getStartTime(),
+ anyOfOriginalEvents.getEndTime());
+
+ this.dataRegionId = String.valueOf(committerKey.getRegionId());
+ this.originFilePaths =
+ originalEvents.stream()
+ .map(PipeTsFileInsertionEvent::getTsFile)
+ .map(File::getPath)
+ .collect(Collectors.toSet());
+ this.commitIds =
+ originalEvents.stream()
+ .map(PipeTsFileInsertionEvent::getCommitId)
+ .distinct()
+ .collect(Collectors.toList());
+
+ // init fields of EnrichedEvent
+ this.committerKey = committerKey;
+ isPatternParsed = bindIsPatternParsed(originalEvents);
+ isTimeParsed = bindIsTimeParsed(originalEvents);
+ this.shouldReportOnCommit = shouldReportProgress;
+
+ // init fields of PipeTsFileInsertionEvent
+ flushPointCount = bindFlushPointCount(originalEvents);
+ overridingProgressIndex = bindOverridingProgressIndex(originalEvents);
+ }
+
+ private static boolean bindIsWithMod(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().anyMatch(PipeTsFileInsertionEvent::isWithMod);
+ }
+
+ private static boolean bindIsLoaded(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().anyMatch(PipeTsFileInsertionEvent::isLoaded);
+ }
+
+ private static boolean bindIsGeneratedByHistoricalExtractor(
+ Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream()
+ .anyMatch(PipeTsFileInsertionEvent::isGeneratedByHistoricalExtractor);
+ }
+
+ private static boolean bindIsTimeParsed(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().noneMatch(EnrichedEvent::shouldParseTime);
+ }
+
+ private static boolean bindIsPatternParsed(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().noneMatch(EnrichedEvent::shouldParsePattern);
+ }
+
+ private static long bindFlushPointCount(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream()
+ .mapToLong(
+ e ->
+ e.getFlushPointCount() == TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET
+ ? 0
+ : e.getFlushPointCount())
+ .sum();
+ }
+
+ private ProgressIndex bindOverridingProgressIndex(Set<PipeTsFileInsertionEvent> originalEvents) {
+ ProgressIndex overridingProgressIndex = MinimumProgressIndex.INSTANCE;
+ for (PipeTsFileInsertionEvent originalEvent : originalEvents) {
+ if (originalEvent.overridingProgressIndex != null) {
+ overridingProgressIndex =
+ overridingProgressIndex.updateToMinimumEqualOrIsAfterProgressIndex(
+ originalEvent.overridingProgressIndex);
+ }
+ }
+ return overridingProgressIndex != null
+ && !overridingProgressIndex.equals(MinimumProgressIndex.INSTANCE)
+ ? overridingProgressIndex
+ : null;
+ }
+
+ @Override
+ public int getRebootTimes() {
+ throw new UnsupportedOperationException(
+ "PipeCompactedTsFileInsertionEvent does not support getRebootTimes.");
+ }
+
+ @Override
+ public boolean hasMultipleCommitIds() {
+ return true;
+ }
+
+ @Override
+ public long getCommitId() {
+ // max of commitIds is used as the commit id for this event
+ return commitIds.stream()
+ .max(Long::compareTo)
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "No commit IDs found in PipeCompactedTsFileInsertionEvent."));
+ }
+
+ // return dummy events for each commit ID (except the max one)
+ @Override
+ public List<EnrichedEvent> getDummyEventsForCommitIds() {
+ return commitIds.stream()
+ .filter(commitId -> commitId != getCommitId())
+ .map(PipeCompactedTsFileInsertionDummyEvent::new)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<Long> getCommitIds() {
+ return commitIds;
+ }
+
+ @Override
+ public boolean equalsInPipeConsensus(final Object o) {
+ throw new UnsupportedOperationException(
+ "PipeCompactedTsFileInsertionEvent does not support equalsInPipeConsensus.");
+ }
+
+ @Override
+ public void eliminateProgressIndex() {
+ if (Objects.isNull(overridingProgressIndex)) {
+ for (final String originFilePath : originFilePaths) {
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(dataRegionId, pipeName, originFilePath);
+ }
+ }
+ }
+
+ public class PipeCompactedTsFileInsertionDummyEvent extends EnrichedEvent {
+
+ private final long commitId;
+
+ public PipeCompactedTsFileInsertionDummyEvent(final long commitId) {
+ super(
+ PipeCompactedTsFileInsertionEvent.this.pipeName,
+ PipeCompactedTsFileInsertionEvent.this.creationTime,
+ PipeCompactedTsFileInsertionEvent.this.pipeTaskMeta,
+ null, // PipePattern is not needed for dummy event
+ Long.MIN_VALUE,
+ Long.MAX_VALUE);
+ this.commitId = commitId; // Use the commitId passed in
+ this.shouldReportOnCommit = false; // Dummy events do not report progress
+ }
+
+ @Override
+ public long getCommitId() {
+ return commitId;
+ }
+
+ @Override
+ public boolean internallyIncreaseResourceReferenceCount(String holderMessage) {
+ return true;
+ }
+
+ @Override
+ public boolean internallyDecreaseResourceReferenceCount(String holderMessage) {
+ return true;
+ }
+
+ @Override
+ public ProgressIndex getProgressIndex() {
+ return MinimumProgressIndex.INSTANCE;
+ }
+
+ @Override
+ public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+ String pipeName,
+ long creationTime,
+ PipeTaskMeta pipeTaskMeta,
+ PipePattern pattern,
+ long startTime,
+ long endTime) {
+ return null;
+ }
+
+ @Override
+ public boolean isGeneratedByPipe() {
+ return false;
+ }
+
+ @Override
+ public boolean mayEventTimeOverlappedWithTimeRange() {
+ return false;
+ }
+
+ @Override
+ public boolean mayEventPathsOverlappedWithPattern() {
+ return false;
+ }
+
+ @Override
+ public String coreReportMessage() {
+ return "PipeCompactedTsFileInsertionDummyEvent";
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index 41931131eb3..de5281976d3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -65,48 +65,37 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileInsertionEvent.class);
- private final TsFileResource resource;
- private File tsFile;
- private long extractTime = 0;
+ protected final TsFileResource resource;
+ protected File tsFile;
+ protected long extractTime = 0;
// This is true iff the modFile exists and should be transferred
- private boolean isWithMod;
- private File modFile;
+ protected boolean isWithMod;
+ protected File modFile;
- private final boolean isLoaded;
- private final boolean isGeneratedByPipe;
- private final boolean isGeneratedByPipeConsensus;
- private final boolean isGeneratedByHistoricalExtractor;
+ protected final boolean isLoaded;
+ protected final boolean isGeneratedByPipe;
+ protected final boolean isGeneratedByPipeConsensus;
+ protected final boolean isGeneratedByHistoricalExtractor;
- private final AtomicBoolean isClosed;
- private final AtomicReference<TsFileInsertionDataContainer> dataContainer;
+ protected final AtomicBoolean isClosed;
+ protected final AtomicReference<TsFileInsertionDataContainer> dataContainer;
// The point count of the TsFile. Used for metrics on PipeConsensus' receiver side.
// May be updated after it is flushed. Should be negative if not set.
- private long flushPointCount = TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET;
+ protected long flushPointCount = TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET;
- private volatile ProgressIndex overridingProgressIndex;
+ protected volatile ProgressIndex overridingProgressIndex;
- public PipeTsFileInsertionEvent(
- final TsFileResource resource,
- final boolean isLoaded,
- final boolean isGeneratedByHistoricalExtractor) {
+ public PipeTsFileInsertionEvent(final TsFileResource resource, final boolean isLoaded) {
// The modFile must be copied before the event is assigned to the listening pipes
this(
- resource,
- true,
- isLoaded,
- isGeneratedByHistoricalExtractor,
- null,
- 0,
- null,
- null,
- Long.MIN_VALUE,
- Long.MAX_VALUE);
+ resource, null, true, isLoaded, false, null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE);
}
public PipeTsFileInsertionEvent(
final TsFileResource resource,
+ final File tsFile,
final boolean isWithMod,
final boolean isLoaded,
final boolean isGeneratedByHistoricalExtractor,
@@ -117,9 +106,13 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
final long startTime,
final long endTime) {
super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime);
-
this.resource = resource;
- tsFile = resource.getTsFile();
+
+ // For events created at assigner or historical extractor, the tsFile is get from the resource
+ // For events created for source, the tsFile is inherited from the assigner, because the
+ // original tsFile may be gone, and we need to get the assigner's hard-linked tsFile to
+ // hard-link it to each pipe dir
+ this.tsFile = Objects.isNull(tsFile) ? resource.getTsFile() : tsFile;
final ModificationFile modFile = resource.getModFile();
this.isWithMod = isWithMod && modFile.exists();
@@ -130,6 +123,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
this.isGeneratedByPipeConsensus = resource.isGeneratedByPipeConsensus();
this.isGeneratedByHistoricalExtractor = isGeneratedByHistoricalExtractor;
+ this.dataContainer = new AtomicReference<>(null);
+
isClosed = new AtomicBoolean(resource.isClosed());
// Register close listener if TsFile is not closed
if (!isClosed.get()) {
@@ -165,8 +160,6 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
// If the status is "closed", then the resource status is "closed", the tsFile won't be altered
// and can be sent.
isClosed.set(resource.isClosed());
-
- this.dataContainer = new AtomicReference<>(null);
}
/**
@@ -174,6 +167,10 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
* otherwise.
*/
public boolean waitForTsFileClose() throws InterruptedException {
+ if (Objects.isNull(resource)) {
+ return true;
+ }
+
if (!isClosed.get()) {
isClosed.set(resource.isClosed());
@@ -204,6 +201,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return !resource.isEmpty();
}
+ @Override
public File getTsFile() {
return tsFile;
}
@@ -226,10 +224,6 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return isLoaded;
}
- public long getFileStartTime() {
- return resource.getFileStartTime();
- }
-
/**
* Only used for metrics on PipeConsensus' receiver side. If the event is recovered after data
* node's restart, the flushPointCount can be not set. It's totally fine for the PipeConsensus'
@@ -252,9 +246,10 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
extractTime = System.nanoTime();
try {
- tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, resource);
+ tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, pipeName);
if (isWithMod) {
- modFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, null);
+ modFile =
+ PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, pipeName);
}
return true;
} catch (final Exception e) {
@@ -275,9 +270,9 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
try {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName);
if (isWithMod) {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName);
}
close();
return true;
@@ -303,24 +298,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public ProgressIndex getProgressIndex() {
- try {
- if (!waitForTsFileClose()) {
- LOGGER.warn(
- "Skipping temporary TsFile {}'s progressIndex, will report MinimumProgressIndex",
- tsFile);
- return MinimumProgressIndex.INSTANCE;
- }
- if (Objects.nonNull(overridingProgressIndex)) {
- return overridingProgressIndex;
- }
- return resource.getMaxProgressIndexAfterClose();
- } catch (final InterruptedException e) {
- LOGGER.warn(
- String.format(
- "Interrupted when waiting for closing TsFile %s.", resource.getTsFilePath()));
- Thread.currentThread().interrupt();
- return MinimumProgressIndex.INSTANCE;
- }
+ return resource.getMaxProgressIndex();
}
/**
@@ -346,9 +324,9 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
}
public void eliminateProgressIndex() {
- if (Objects.isNull(overridingProgressIndex)) {
+ if (Objects.isNull(overridingProgressIndex) && Objects.nonNull(resource)) {
PipeTsFileEpochProgressIndexKeeper.getInstance()
- .eliminateProgressIndex(resource.getDataRegionId(), resource.getTsFilePath());
+ .eliminateProgressIndex(resource.getDataRegionId(), pipeName, resource.getTsFilePath());
}
}
@@ -362,6 +340,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
final long endTime) {
return new PipeTsFileInsertionEvent(
resource,
+ tsFile,
isWithMod,
isLoaded,
isGeneratedByHistoricalExtractor,
@@ -380,16 +359,15 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public boolean mayEventTimeOverlappedWithTimeRange() {
- // If the tsFile is not closed the resource.getFileEndTime() will be Long.MIN_VALUE
- // In that case we only judge the resource.getFileStartTime() to avoid losing data
- return isClosed.get()
- ? startTime <= resource.getFileEndTime() && resource.getFileStartTime() <= endTime
- : resource.getFileStartTime() <= endTime;
+ // Notice that this is only called at realtime extraction, and the tsFile is always closed
+ // Thus we can use the end time to judge the overlap
+ return Objects.isNull(resource)
+ || startTime <= resource.getFileEndTime() && resource.getFileStartTime() <= endTime;
}
@Override
public boolean mayEventPathsOverlappedWithPattern() {
- if (!resource.isClosed()) {
+ if (Objects.isNull(resource) || !resource.isClosed()) {
return true;
}
@@ -397,7 +375,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
final Map<IDeviceID, Boolean> deviceIsAlignedMap =
PipeDataNodeResourceManager.tsfile()
.getDeviceIsAlignedMapFromCache(
- PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()),
+ PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ resource.getTsFile(), pipeName),
false);
final Set<IDeviceID> deviceSet =
Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices();
@@ -569,7 +548,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
} catch (final IOException e) {
close();
- final String errorMsg = String.format("Read TsFile %s error.", resource.getTsFilePath());
+ final String errorMsg = String.format("Read TsFile %s error.", tsFile.getPath());
LOGGER.warn(errorMsg, e);
throw new PipeException(errorMsg);
}
@@ -617,8 +596,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public String toString() {
return String.format(
- "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, isClosed=%s, dataContainer=%s}",
- resource, tsFile, isLoaded, isGeneratedByPipe, isClosed.get(), dataContainer)
+ "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, dataContainer=%s}",
+ resource, tsFile, isLoaded, isGeneratedByPipe, dataContainer)
+ " - "
+ super.toString();
}
@@ -626,8 +605,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public String coreReportMessage() {
return String.format(
- "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, isClosed=%s}",
- resource, tsFile, isLoaded, isGeneratedByPipe, isClosed.get())
+ "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s}",
+ resource, tsFile, isLoaded, isGeneratedByPipe)
+ " - "
+ super.coreReportMessage();
}
@@ -644,6 +623,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return new PipeTsFileInsertionEventResource(
this.isReleased,
this.referenceCount,
+ this.pipeName,
this.tsFile,
this.isWithMod,
this.modFile,
@@ -656,15 +636,18 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
private final boolean isWithMod;
private final File modFile;
private final AtomicReference<TsFileInsertionDataContainer> dataContainer;
+ private final String pipeName;
private PipeTsFileInsertionEventResource(
final AtomicBoolean isReleased,
final AtomicInteger referenceCount,
+ final String pipeName,
final File tsFile,
final boolean isWithMod,
final File modFile,
final AtomicReference<TsFileInsertionDataContainer> dataContainer) {
super(isReleased, referenceCount);
+ this.pipeName = pipeName;
this.tsFile = tsFile;
this.isWithMod = isWithMod;
this.modFile = modFile;
@@ -675,9 +658,9 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
protected void finalizeResource() {
try {
// decrease reference count
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName);
if (isWithMod) {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName);
}
// close data container
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
index 21ce698141d..1050950d176 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
@@ -28,7 +28,8 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.container.query.TsFileInsert
import org.apache.iotdb.db.pipe.event.common.tsfile.container.scan.TsFileInsertionScanDataContainer;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResource;
+import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFilePublicResource;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.PlainDeviceID;
@@ -79,8 +80,8 @@ public class TsFileInsertionDataContainerProvider {
// Use scan container to save memory
if ((double) PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes()
- / PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- > PipeTsFileResource.MEMORY_SUFFICIENT_THRESHOLD) {
+ / PipeMemoryManager.getTotalNonFloatingMemorySizeInBytes()
+ > PipeTsFilePublicResource.MEMORY_SUFFICIENT_THRESHOLD) {
return new TsFileInsertionScanDataContainer(
pipeName, creationTime, tsFile, pattern, startTime, endTime, pipeTaskMeta, sourceEvent);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
index ef9db9a4cb0..0c3bce5c399 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
@@ -28,7 +28,6 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch.TsFileEpochM
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
public class PipeRealtimeEventFactory {
@@ -37,22 +36,13 @@ public class PipeRealtimeEventFactory {
public static PipeRealtimeEvent createRealtimeEvent(
final TsFileResource resource, final boolean isLoaded) {
return TS_FILE_EPOCH_MANAGER.bindPipeTsFileInsertionEvent(
- new PipeTsFileInsertionEvent(resource, isLoaded, false), resource);
+ new PipeTsFileInsertionEvent(resource, isLoaded), resource);
}
public static PipeRealtimeEvent createRealtimeEvent(
- final WALEntryHandler walEntryHandler,
- final InsertNode insertNode,
- final TsFileResource resource) {
+ final InsertNode insertNode, final TsFileResource resource) {
return TS_FILE_EPOCH_MANAGER.bindPipeInsertNodeTabletInsertionEvent(
- new PipeInsertNodeTabletInsertionEvent(
- walEntryHandler,
- insertNode.getDevicePath(),
- insertNode.getProgressIndex(),
- insertNode.isAligned(),
- insertNode.isGeneratedByPipe()),
- insertNode,
- resource);
+ new PipeInsertNodeTabletInsertionEvent(insertNode), insertNode, resource);
}
public static PipeRealtimeEvent createRealtimeEvent(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
index a8d95d44810..612b1a9701f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
@@ -48,10 +48,6 @@ public class DataRegionWatermarkInjector {
return injectionIntervalInMs;
}
- public long getNextInjectionTime() {
- return nextInjectionTime;
- }
-
public PipeWatermarkEvent inject() {
if (System.currentTimeMillis() < nextInjectionTime) {
return null;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
index aa2f849f56f..8158c22d1a6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.pipe.extractor.dataregion;
import org.apache.iotdb.commons.consensus.DataRegionId;
-import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBPipePattern;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
import org.apache.iotdb.commons.pipe.extractor.IoTDBExtractor;
@@ -38,7 +38,6 @@ import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
@@ -47,11 +46,12 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.apache.iotdb.pipe.api.exception.PipeException;
-import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+
import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference;
@@ -95,13 +95,12 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionExtractor.class);
- private PipeHistoricalDataRegionExtractor historicalExtractor;
+ private @Nullable PipeHistoricalDataRegionExtractor historicalExtractor;
private PipeRealtimeDataRegionExtractor realtimeExtractor;
private DataRegionWatermarkInjector watermarkInjector;
private boolean hasNoExtractionNeed = true;
- private boolean shouldExtractInsertion = false;
private boolean shouldExtractDeletion = false;
@Override
@@ -116,7 +115,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
return;
}
hasNoExtractionNeed = false;
- shouldExtractInsertion = insertionDeletionListeningOptionPair.getLeft();
shouldExtractDeletion = insertionDeletionListeningOptionPair.getRight();
if (insertionDeletionListeningOptionPair.getLeft().equals(true)
@@ -213,10 +211,22 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
EXTRACTOR_HISTORY_END_TIME_KEY);
}
- constructHistoricalExtractor();
+ if (validator
+ .getParameters()
+ .getBooleanOrDefault(SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || validator
+ .getParameters()
+ .getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE)) {
+ // Do not flush or open historical extractor when historical tsFile is disabled
+ constructHistoricalExtractor();
+ }
constructRealtimeExtractor(validator.getParameters());
- historicalExtractor.validate(validator);
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.validate(validator);
+ }
realtimeExtractor.validate(validator);
}
@@ -237,12 +247,10 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
private void constructHistoricalExtractor() {
- // Enable historical extractor by default
historicalExtractor = new PipeHistoricalDataRegionTsFileExtractor();
}
- private void constructRealtimeExtractor(final PipeParameters parameters)
- throws IllegalPathException {
+ private void constructRealtimeExtractor(final PipeParameters parameters) {
// Use heartbeat only extractor if disable realtime extractor
if (!parameters.getBooleanOrDefault(
Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
@@ -270,7 +278,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
// Use hybrid mode by default
if (!parameters.hasAnyAttributes(EXTRACTOR_REALTIME_MODE_KEY, SOURCE_REALTIME_MODE_KEY)) {
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
LOGGER.info(
"Pipe: '{}' is not set, use hybrid mode by default.", EXTRACTOR_REALTIME_MODE_KEY);
@@ -285,15 +292,12 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
case EXTRACTOR_REALTIME_MODE_HYBRID_VALUE:
case EXTRACTOR_REALTIME_MODE_LOG_VALUE:
case EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
break;
case EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionLogExtractor();
break;
default:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
if (LOGGER.isWarnEnabled()) {
LOGGER.warn(
@@ -303,28 +307,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
}
- private void checkWalEnableAndSetUncompressed(final PipeParameters parameters)
- throws IllegalPathException {
- if (Boolean.TRUE.equals(
- DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters)
- .getLeft())
- && IoTDBDescriptor.getInstance().getConfig().getWalMode().equals(WALMode.DISABLE)) {
- throw new PipeException(
- "The pipe cannot transfer realtime insertion if data region disables wal. Please set 'realtime.mode'='batch' in source parameters when enabling realtime transmission.");
- }
-
- if (!IoTDBDescriptor.getInstance()
- .getConfig()
- .getWALCompressionAlgorithm()
- .equals(CompressionType.UNCOMPRESSED)) {
- LOGGER.info(
- "The pipe prefers uncompressed wal, and may introduce certain delay in realtime insert syncing without it. Hence, we change it to uncompressed.");
- IoTDBDescriptor.getInstance()
- .getConfig()
- .setWALCompressionAlgorithm(CompressionType.UNCOMPRESSED);
- }
- }
-
@Override
public void customize(
final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration)
@@ -335,7 +317,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
super.customize(parameters, configuration);
- historicalExtractor.customize(parameters, configuration);
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.customize(parameters, configuration);
+ }
realtimeExtractor.customize(parameters, configuration);
// Set watermark injector
@@ -372,7 +356,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
"Pipe {}@{}: Starting historical extractor {} and realtime extractor {}.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName());
super.start();
@@ -407,7 +393,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
"Pipe {}@{}: Started historical extractor {} and realtime extractor {} successfully within {} ms.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName(),
System.currentTimeMillis() - startTime);
return;
@@ -425,14 +413,18 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
// There can still be writing when tsFile events are added. If we start
// realtimeExtractor after the process, then this part of data will be lost.
realtimeExtractor.start();
- historicalExtractor.start();
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.start();
+ }
} catch (final Exception e) {
exceptionHolder.set(e);
LOGGER.warn(
"Pipe {}@{}: Start historical extractor {} and realtime extractor {} error.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName(),
e);
}
@@ -451,7 +443,7 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
Event event = null;
- if (!historicalExtractor.hasConsumedAll()) {
+ if (Objects.nonNull(historicalExtractor) && !historicalExtractor.hasConsumedAll()) {
event = historicalExtractor.supply();
} else {
if (Objects.nonNull(watermarkInjector)) {
@@ -481,32 +473,21 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
return;
}
- historicalExtractor.close();
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.close();
+ }
realtimeExtractor.close();
if (Objects.nonNull(taskID)) {
PipeDataRegionExtractorMetrics.getInstance().deregister(taskID);
}
}
- //////////////////////////// APIs provided for detecting stuck ////////////////////////////
-
- public boolean shouldExtractInsertion() {
- return shouldExtractInsertion;
- }
-
- public boolean isStreamMode() {
- return realtimeExtractor instanceof PipeRealtimeDataRegionHybridExtractor
- || realtimeExtractor instanceof PipeRealtimeDataRegionLogExtractor;
- }
-
- public boolean hasConsumedAllHistoricalTsFiles() {
- return historicalExtractor.hasConsumedAll();
- }
-
//////////////////////////// APIs provided for metric framework ////////////////////////////
public int getHistoricalTsFileInsertionEventCount() {
- return hasBeenStarted.get() ? historicalExtractor.getPendingQueueSize() : 0;
+ return hasBeenStarted.get() && Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getPendingQueueSize()
+ : 0;
}
public int getTabletInsertionEventCount() {
@@ -520,10 +501,4 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
public int getPipeHeartbeatEventCount() {
return hasBeenStarted.get() ? realtimeExtractor.getPipeHeartbeatEventCount() : 0;
}
-
- public int getEventCount() {
- return hasBeenStarted.get()
- ? (historicalExtractor.getPendingQueueSize() + realtimeExtractor.getEventCount())
- : 0;
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
index 3ea98cf5bb3..41cc715eb52 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -33,9 +33,9 @@ import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
@@ -57,10 +57,10 @@ import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
@@ -94,9 +94,6 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
private static final Logger LOGGER =
LoggerFactory.getLogger(PipeHistoricalDataRegionTsFileExtractor.class);
- private static final Map<Integer, Long> DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP = new HashMap<>();
- private static final long PIPE_MIN_FLUSH_INTERVAL_IN_MS = 2000;
-
private String pipeName;
private long creationTime;
@@ -219,17 +216,14 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
try {
historicalDataExtractionStartTime =
- isHistoricalExtractorEnabled
- && parameters.hasAnyAttributes(
- EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY)
+ parameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY)
? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
parameters.getStringByKeys(
EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY))
: Long.MIN_VALUE;
historicalDataExtractionEndTime =
- isHistoricalExtractorEnabled
- && parameters.hasAnyAttributes(
- EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)
+ parameters.hasAnyAttributes(EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)
? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
parameters.getStringByKeys(
EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY))
@@ -273,10 +267,6 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
startIndex = environment.getPipeTaskMeta().restoreProgressIndex();
dataRegionId = environment.getRegionId();
- synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) {
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.putIfAbsent(dataRegionId, 0L);
- }
-
pipePattern = PipePattern.parsePipePatternFromSourceParameters(parameters);
final DataRegion dataRegion =
@@ -300,34 +290,6 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// PipeHistoricalDataRegionExtractor from implementation perspective.
: environment.getCreationTime();
- // Only invoke flushDataRegionAllTsFiles() when the pipe runs in the realtime only mode.
- // realtime only mode -> (historicalDataExtractionTimeLowerBound != Long.MIN_VALUE)
- //
- // Ensure that all data in the data region is flushed to disk before extracting data.
- // This ensures the generation time of all newly generated TsFiles (realtime data) after the
- // invocation of flushDataRegionAllTsFiles() is later than the creationTime of the pipe
- // (historicalDataExtractionTimeLowerBound).
- //
- // Note that: the generation time of the TsFile is the time when the TsFile is created, not
- // the time when the data is flushed to the TsFile.
- //
- // Then we can use the generation time of the TsFile to determine whether the data in the
- // TsFile should be extracted by comparing the generation time of the TsFile with the
- // historicalDataExtractionTimeLowerBound when starting the pipe in realtime only mode.
- //
- // If we don't invoke flushDataRegionAllTsFiles() in the realtime only mode, the data generated
- // between the creation time of the pipe the time when the pipe starts will be lost.
- if (historicalDataExtractionTimeLowerBound != Long.MIN_VALUE) {
- synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) {
- final long lastFlushedByPipeTime =
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.get(dataRegionId);
- if (System.currentTimeMillis() - lastFlushedByPipeTime >= PIPE_MIN_FLUSH_INTERVAL_IN_MS) {
- flushDataRegionAllTsFiles();
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.replace(dataRegionId, System.currentTimeMillis());
- }
- }
- }
-
shouldTransferModFile =
parameters.getBooleanOrDefault(
Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY),
@@ -357,24 +319,9 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
}
}
- private void flushDataRegionAllTsFiles() {
- final DataRegion dataRegion =
- StorageEngine.getInstance().getDataRegion(new DataRegionId(dataRegionId));
- if (Objects.isNull(dataRegion)) {
- return;
- }
-
- dataRegion.writeLock("Pipe: create historical TsFile extractor");
- try {
- dataRegion.syncCloseAllWorkingTsFileProcessors();
- } finally {
- dataRegion.writeUnlock();
- }
- }
-
@Override
public synchronized void start() {
- if (!shouldExtractInsertion) {
+ if (!shouldExtractInsertion || !isHistoricalExtractorEnabled) {
hasBeenStarted = true;
return;
}
@@ -406,48 +353,29 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// consensus pipe, and the lastFlushed timestamp is not updated here.
if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) {
dataRegion.syncCloseAllWorkingTsFileProcessors();
- LOGGER.info(
- "Pipe {}@{}: finish to flush data region, took {} ms",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - startHistoricalExtractionTime);
} else {
- synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) {
- final long lastFlushedByPipeTime =
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.get(dataRegionId);
- if (System.currentTimeMillis() - lastFlushedByPipeTime >= PIPE_MIN_FLUSH_INTERVAL_IN_MS) {
- dataRegion.syncCloseAllWorkingTsFileProcessors();
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.replace(
- dataRegionId, System.currentTimeMillis());
- LOGGER.info(
- "Pipe {}@{}: finish to flush data region, took {} ms",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - startHistoricalExtractionTime);
- } else {
- LOGGER.info(
- "Pipe {}@{}: skip to flush data region, last flushed time {} ms ago",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - lastFlushedByPipeTime);
- }
- }
+ dataRegion.asyncCloseAllWorkingTsFileProcessors();
}
+ LOGGER.info(
+ "Pipe {}@{}: finish to flush data region, took {} ms",
+ pipeName,
+ dataRegionId,
+ System.currentTimeMillis() - startHistoricalExtractionTime);
final TsFileManager tsFileManager = dataRegion.getTsFileManager();
tsFileManager.readLock();
try {
final int originalSequenceTsFileCount = tsFileManager.size(true);
- final int originalUnsequenceTsFileCount = tsFileManager.size(false);
+ final int originalUnSequenceTsFileCount = tsFileManager.size(false);
final List<TsFileResource> resourceList =
- new ArrayList<>(originalSequenceTsFileCount + originalUnsequenceTsFileCount);
+ new ArrayList<>(originalSequenceTsFileCount + originalUnSequenceTsFileCount);
LOGGER.info(
"Pipe {}@{}: start to extract historical TsFile, original sequence file count {}, "
- + "original unsequence file count {}, start progress index {}",
+ + "original unSequence file count {}, start progress index {}",
pipeName,
dataRegionId,
originalSequenceTsFileCount,
- originalUnsequenceTsFileCount,
+ originalUnSequenceTsFileCount,
startIndex);
final Collection<TsFileResource> sequenceTsFileResources =
@@ -460,9 +388,13 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// not transfer pipe requests.
&& (!resource.isGeneratedByPipe() || isForwardingPipeRequests)
&& (
- // Some resource may not be closed due to the control of
- // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+ // If the tsFile is not already marked closing, it is not captured by
+ // the pipe realtime module. Thus, we can wait for the realtime sync
+ // module to handle this, to avoid blocking the pipe sync process.
!resource.isClosed()
+ && Optional.ofNullable(resource.getProcessor())
+ .map(TsFileProcessor::alreadyMarkedClosing)
+ .orElse(true)
|| mayTsFileContainUnprocessedData(resource)
&& isTsFileResourceOverlappedWithTimeRange(resource)
&& isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
@@ -470,7 +402,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
.collect(Collectors.toList());
resourceList.addAll(sequenceTsFileResources);
- final Collection<TsFileResource> unsequenceTsFileResources =
+ final Collection<TsFileResource> unSequenceTsFileResources =
tsFileManager.getTsFileList(false).stream()
.filter(
resource ->
@@ -480,15 +412,19 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// not transfer pipe requests.
&& (!resource.isGeneratedByPipe() || isForwardingPipeRequests)
&& (
- // Some resource may not be closed due to the control of
- // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+ // If the tsFile is not already marked closing, it is not captured by
+ // the pipe realtime module. Thus, we can wait for the realtime sync
+ // module to handle this, to avoid blocking the pipe sync process.
!resource.isClosed()
+ && Optional.ofNullable(resource.getProcessor())
+ .map(TsFileProcessor::alreadyMarkedClosing)
+ .orElse(true)
|| mayTsFileContainUnprocessedData(resource)
&& isTsFileResourceOverlappedWithTimeRange(resource)
&& isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
&& mayTsFileResourceOverlappedWithPattern(resource)))
.collect(Collectors.toList());
- resourceList.addAll(unsequenceTsFileResources);
+ resourceList.addAll(unSequenceTsFileResources);
resourceList.removeIf(
resource -> {
@@ -496,7 +432,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// Will unpin it after the PipeTsFileInsertionEvent is created and pinned.
try {
PipeDataNodeResourceManager.tsfile()
- .pinTsFileResource(resource, shouldTransferModFile);
+ .pinTsFileResource(resource, shouldTransferModFile, pipeName);
return false;
} catch (final IOException e) {
LOGGER.warn("Pipe: failed to pin TsFileResource {}", resource.getTsFilePath(), e);
@@ -518,10 +454,10 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
dataRegionId,
sequenceTsFileResources.size(),
originalSequenceTsFileCount,
- unsequenceTsFileResources.size(),
- originalUnsequenceTsFileCount,
+ unSequenceTsFileResources.size(),
+ originalUnSequenceTsFileCount,
resourceList.size(),
- originalSequenceTsFileCount + originalUnsequenceTsFileCount,
+ originalSequenceTsFileCount + originalUnSequenceTsFileCount,
System.currentTimeMillis() - startHistoricalExtractionTime);
} finally {
tsFileManager.readUnlock();
@@ -540,8 +476,8 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
if (startIndex instanceof StateProgressIndex) {
startIndex = ((StateProgressIndex) startIndex).getInnerProgressIndex();
}
- return !startIndex.isAfter(resource.getMaxProgressIndexAfterClose())
- && !startIndex.equals(resource.getMaxProgressIndexAfterClose());
+ return !startIndex.isAfter(resource.getMaxProgressIndex())
+ && !startIndex.equals(resource.getMaxProgressIndex());
}
private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) {
@@ -549,9 +485,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
try {
final Map<IDeviceID, Boolean> deviceIsAlignedMap =
PipeDataNodeResourceManager.tsfile()
- .getDeviceIsAlignedMapFromCache(
- PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()),
- false);
+ .getDeviceIsAlignedMapFromCache(resource.getTsFile(), false);
deviceSet =
Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices();
} catch (final IOException e) {
@@ -609,6 +543,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
}
final TsFileResource resource = pendingQueue.poll();
+
if (resource == null) {
final PipeTerminateEvent terminateEvent =
new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, dataRegionId);
@@ -627,6 +562,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
final PipeTsFileInsertionEvent event =
new PipeTsFileInsertionEvent(
resource,
+ null,
shouldTransferModFile,
false,
true,
@@ -657,7 +593,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
return isReferenceCountIncreased ? event : null;
} finally {
try {
- PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource);
+ PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName);
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after creating event, original path: {}",
@@ -687,7 +623,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
pendingQueue.forEach(
resource -> {
try {
- PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource);
+ PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName);
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after dropping pipe, original path: {}",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
index c7a778357a4..0eaa94e0262 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.pipe.extractor.dataregion.realtime;
import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
@@ -31,8 +32,11 @@ import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeTimePartitionListener;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
@@ -395,6 +399,13 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
}
}
+ protected void maySkipIndex4Event(final PipeRealtimeEvent event) {
+ if (event.getEvent() instanceof PipeTsFileInsertionEvent
+ || event.getEvent() instanceof PipeInsertNodeTabletInsertionEvent) {
+ maySkipProgressIndexForRealtimeEvent(event);
+ }
+ }
+
protected Event supplyHeartbeat(final PipeRealtimeEvent event) {
if (event.increaseReferenceCount(PipeRealtimeDataRegionExtractor.class.getName())) {
return event.getEvent();
@@ -496,6 +507,30 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
return shouldTransferModFile;
}
+ private void maySkipProgressIndexForRealtimeEvent(final PipeRealtimeEvent event) {
+ if (PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .isProgressIndexAfterOrEquals(
+ dataRegionId,
+ pipeName,
+ event.getTsFileEpoch().getFilePath(),
+ getProgressIndex4RealtimeEvent(event))) {
+ event.skipReportOnCommit();
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Pipe {} on data region {} skip commit of event {} because it was flushed prematurely.",
+ pipeName,
+ dataRegionId,
+ event.coreReportMessage());
+ }
+ }
+ }
+
+ private ProgressIndex getProgressIndex4RealtimeEvent(final PipeRealtimeEvent event) {
+ return event.getEvent() instanceof PipeTsFileInsertionEvent
+ ? ((PipeTsFileInsertionEvent) event.getEvent()).forceGetProgressIndex()
+ : event.getProgressIndex();
+ }
+
@Override
public String toString() {
return "PipeRealtimeDataRegionExtractor{"
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
index 500a195e600..51bd3819f54 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
@@ -31,12 +30,9 @@ import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch.TsFileEpoch;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
-import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
@@ -45,7 +41,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Objects;
-import java.util.function.Consumer;
public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegionExtractor {
@@ -87,6 +82,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
if (canNotUseTabletAnyMore(event)) {
event.getTsFileEpoch().migrateState(this, curState -> TsFileEpoch.State.USING_TSFILE);
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .registerProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getResource());
} else {
event
.getTsFileEpoch()
@@ -145,6 +142,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
}
private void extractTsFileInsertion(final PipeRealtimeEvent event) {
+ // Notice that, if the tsFile is partially extracted because the pipe is not opened before, the
+ // former data won't be extracted
event
.getTsFileEpoch()
.migrateState(
@@ -152,34 +151,13 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
state -> {
switch (state) {
case EMPTY:
+ return ((PipeTsFileInsertionEvent) event.getEvent()).isLoaded()
+ ? TsFileEpoch.State.USING_TSFILE
+ : TsFileEpoch.State.USING_TABLET;
+ case USING_TABLET:
+ return TsFileEpoch.State.USING_TABLET;
case USING_TSFILE:
return TsFileEpoch.State.USING_TSFILE;
- case USING_TABLET:
- if (((PipeTsFileInsertionEvent) event.getEvent()).getFileStartTime()
- < event.getTsFileEpoch().getInsertNodeMinTime()) {
- // Some insert nodes in the tsfile epoch are not captured by pipe, so we should
- // capture the tsfile event to make sure all data in the tsfile epoch can be
- // extracted.
- //
- // The situation can be caused by the following operations:
- // 1. PipeA: start historical data extraction with flush
- // 2. Data insertion
- // 3. PipeB: start realtime data extraction
- // 4. PipeB: start historical data extraction without flush
- // 5. Data inserted in the step2 is not captured by PipeB, and if its tsfile
- // epoch's state is USING_TABLET, the tsfile event will be ignored, which
- // will cause the data loss in the tsfile epoch.
- LOGGER.info(
- "The tsFile {}'s epoch's start time {} is smaller than the captured insertNodes' min time {}, will regard it as data loss or un-sequential, will extract the tsFile",
- ((PipeTsFileInsertionEvent) event.getEvent()).getTsFile(),
- ((PipeTsFileInsertionEvent) event.getEvent()).getFileStartTime(),
- event.getTsFileEpoch().getInsertNodeMinTime());
- return TsFileEpoch.State.USING_BOTH;
- } else {
- // All data in the tsfile epoch has been extracted in tablet mode, so we should
- // simply keep the state of the tsfile epoch and discard the tsfile event.
- return TsFileEpoch.State.USING_TABLET;
- }
case USING_BOTH:
default:
return canNotUseTabletAnyMore(event)
@@ -225,50 +203,13 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
private boolean canNotUseTabletAnyMore(final PipeRealtimeEvent event) {
// In the following 4 cases, we should not extract this tablet event. all the data
// represented by the tablet event should be carried by the following tsfile event:
- // 0. If the remaining insert event count is too large, we need to reduce the accumulated
- // tablets.
- // 1. If Wal size > maximum size of wal buffer,
// the write operation will be throttled, so we should not extract any more tablet events.
- // 2. The shallow memory usage of the insert node has reached the dangerous threshold.
- // 3. Deprecated logics (unused by default)
- return mayRemainingInsertNodeEventExceedLimit(event)
- || mayWalSizeReachThrottleThreshold(event)
- || mayInsertNodeMemoryReachDangerousThreshold(event)
+ // 1. The shallow memory usage of the insert node has reached the dangerous threshold.
+ // 2. Deprecated logics (unused by default)
+ return mayInsertNodeMemoryReachDangerousThreshold(event)
|| canNotUseTabletAnymoreDeprecated(event);
}
- private boolean mayRemainingInsertNodeEventExceedLimit(final PipeRealtimeEvent event) {
- final boolean mayRemainingInsertEventExceedLimit =
- PipeDataNodeSinglePipeMetrics.getInstance().mayRemainingInsertEventExceedLimit(pipeID);
- if (mayRemainingInsertEventExceedLimit && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(0): remaining insert event has reached max allowed insert event count {}",
- pipeName,
- dataRegionId,
- PipeConfig.getInstance().getPipeMaxAllowedRemainingInsertEventCountPerPipe()));
- }
- return mayRemainingInsertEventExceedLimit;
- }
-
- private boolean mayWalSizeReachThrottleThreshold(final PipeRealtimeEvent event) {
- final boolean mayWalSizeReachThrottleThreshold =
- 3 * WALManager.getInstance().getTotalDiskUsage()
- > IoTDBDescriptor.getInstance().getConfig().getThrottleThreshold();
- if (mayWalSizeReachThrottleThreshold && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(1): Wal size {} has reached throttle threshold {}",
- pipeName,
- dataRegionId,
- WALManager.getInstance().getTotalDiskUsage(),
- IoTDBDescriptor.getInstance().getConfig().getThrottleThreshold() / 3.0d));
- }
- return mayWalSizeReachThrottleThreshold;
- }
-
private boolean mayInsertNodeMemoryReachDangerousThreshold(final PipeRealtimeEvent event) {
final long floatingMemoryUsageInByte =
PipeDataNodeAgent.task().getFloatingMemoryUsageInByte(pipeName);
@@ -276,27 +217,19 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
final long totalFloatingMemorySizeInBytes =
PipeMemoryManager.getTotalFloatingMemorySizeInBytes();
final boolean mayInsertNodeMemoryReachDangerousThreshold =
- 3 * floatingMemoryUsageInByte * pipeCount >= 2 * totalFloatingMemorySizeInBytes;
+ floatingMemoryUsageInByte * pipeCount >= totalFloatingMemorySizeInBytes;
if (mayInsertNodeMemoryReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(2): The shallow memory usage of the insert node {} has reached the dangerous threshold {}",
- pipeName,
- dataRegionId,
- floatingMemoryUsageInByte * pipeCount,
- 2 * totalFloatingMemorySizeInBytes / 3.0d));
+ LOGGER.info(
+ "Pipe task {}@{} canNotUseTabletAnyMore(1) for tsFile {}: The memory usage of the insert node {} has reached the dangerous threshold {}",
+ pipeName,
+ dataRegionId,
+ event.getTsFileEpoch().getFilePath(),
+ floatingMemoryUsageInByte * pipeCount,
+ totalFloatingMemorySizeInBytes);
}
return mayInsertNodeMemoryReachDangerousThreshold;
}
- private void logByLogManager(final Consumer<Logger> infoFunction) {
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeRealtimeDataRegionHybridExtractor.class, getTaskID(), Integer.MAX_VALUE, 100, 1)
- .ifPresent(infoFunction);
- }
-
/**
* These judgements are deprecated, and are only reserved for manual operation and compatibility.
*/
@@ -304,54 +237,14 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
private boolean canNotUseTabletAnymoreDeprecated(final PipeRealtimeEvent event) {
// In the following 5 cases, we should not extract any more tablet events. all the data
// represented by the tablet events should be carried by the following tsfile event:
- // 0. If the pipe task is currently restarted.
- // 1. The number of pinned memTables has reached the dangerous threshold.
- // 2. The number of historical tsFile events to transfer has exceeded the limit.
- // 3. The number of realtime tsfile events to transfer has exceeded the limit.
- // 4. The number of linked tsFiles has reached the dangerous threshold.
- return isPipeTaskCurrentlyRestarted(event)
- || mayMemTablePinnedCountReachDangerousThreshold(event)
- || isHistoricalTsFileEventCountExceededLimit(event)
+ // 1. The number of historical tsFile events to transfer has exceeded the limit.
+ // 2. The number of realtime tsfile events to transfer has exceeded the limit.
+ // 3. The number of linked tsFiles has reached the dangerous threshold.
+ return isHistoricalTsFileEventCountExceededLimit(event)
|| isRealtimeTsFileEventCountExceededLimit(event)
|| mayTsFileLinkedCountReachDangerousThreshold(event);
}
- private boolean isPipeTaskCurrentlyRestarted(final PipeRealtimeEvent event) {
- if (!PipeConfig.getInstance().isPipeEpochKeepTsFileAfterStuckRestartEnabled()) {
- return false;
- }
-
- final boolean isPipeTaskCurrentlyRestarted =
- PipeDataNodeAgent.task().isPipeTaskCurrentlyRestarted(pipeName);
- if (isPipeTaskCurrentlyRestarted && event.mayExtractorUseTablets(this)) {
- LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(0): Pipe task is currently restarted",
- pipeName,
- dataRegionId);
- }
- return isPipeTaskCurrentlyRestarted;
- }
-
- private boolean mayMemTablePinnedCountReachDangerousThreshold(final PipeRealtimeEvent event) {
- if (PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount() == Integer.MAX_VALUE) {
- return false;
- }
- final boolean mayMemTablePinnedCountReachDangerousThreshold =
- PipeDataNodeResourceManager.wal().getPinnedWalCount()
- >= PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber();
- if (mayMemTablePinnedCountReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
- LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(1): The number of pinned memTables {} has reached the dangerous threshold {}",
- pipeName,
- dataRegionId,
- PipeDataNodeResourceManager.wal().getPinnedWalCount(),
- PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber());
- }
- return mayMemTablePinnedCountReachDangerousThreshold;
- }
-
private boolean isHistoricalTsFileEventCountExceededLimit(final PipeRealtimeEvent event) {
if (PipeConfig.getInstance().getPipeMaxAllowedHistoricalTsFilePerDataRegion()
== Integer.MAX_VALUE) {
@@ -365,9 +258,10 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
>= PipeConfig.getInstance().getPipeMaxAllowedHistoricalTsFilePerDataRegion();
if (isHistoricalTsFileEventCountExceededLimit && event.mayExtractorUseTablets(this)) {
LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(2): The number of historical tsFile events {} has exceeded the limit {}",
+ "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(1) for tsFile {}: The number of historical tsFile events {} has exceeded the limit {}",
pipeName,
dataRegionId,
+ event.getTsFileEpoch().getFilePath(),
extractor.getHistoricalTsFileInsertionEventCount(),
PipeConfig.getInstance().getPipeMaxAllowedHistoricalTsFilePerDataRegion());
}
@@ -384,9 +278,10 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
>= PipeConfig.getInstance().getPipeMaxAllowedPendingTsFileEpochPerDataRegion();
if (isRealtimeTsFileEventCountExceededLimit && event.mayExtractorUseTablets(this)) {
LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(3): The number of realtime tsFile events {} has exceeded the limit {}",
+ "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(2) for tsFile {}: The number of realtime tsFile events {} has exceeded the limit {}",
pipeName,
dataRegionId,
+ event.getTsFileEpoch().getFilePath(),
pendingQueue.getTsFileInsertionEventCount(),
PipeConfig.getInstance().getPipeMaxAllowedPendingTsFileEpochPerDataRegion());
}
@@ -398,14 +293,15 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
return false;
}
final boolean mayTsFileLinkedCountReachDangerousThreshold =
- PipeDataNodeResourceManager.tsfile().getLinkedTsfileCount()
+ PipeDataNodeResourceManager.tsfile().getLinkedTsFileCount(pipeName)
>= PipeConfig.getInstance().getPipeMaxAllowedLinkedTsFileCount();
if (mayTsFileLinkedCountReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(4): The number of linked tsFiles {} has reached the dangerous threshold {}",
+ "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(3) for tsFile {}: The number of linked tsFiles {} has reached the dangerous threshold {}",
pipeName,
dataRegionId,
- PipeDataNodeResourceManager.tsfile().getLinkedTsfileCount(),
+ event.getTsFileEpoch().getFilePath(),
+ PipeDataNodeResourceManager.tsfile().getLinkedTsFileCount(pipeName),
PipeConfig.getInstance().getPipeMaxAllowedLinkedTsFileCount());
}
return mayTsFileLinkedCountReachDangerousThreshold;
@@ -440,6 +336,7 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
PipeRealtimeDataRegionHybridExtractor.class.getName(), false);
if (suppliedEvent != null) {
+ maySkipIndex4Event(realtimeEvent);
return suppliedEvent;
}
@@ -473,11 +370,14 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
});
final TsFileEpoch.State state = event.getTsFileEpoch().getState(this);
+ if (state == TsFileEpoch.State.USING_TSFILE) {
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .registerProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getResource());
+ }
+
switch (state) {
case USING_TSFILE:
// If the state is USING_TSFILE, discard the event and poll the next one.
- PipeTsFileEpochProgressIndexKeeper.getInstance()
- .eliminateProgressIndex(dataRegionId, event.getTsFileEpoch().getFilePath());
return null;
case EMPTY:
case USING_TABLET:
@@ -518,6 +418,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
switch (state) {
case USING_TABLET:
// If the state is USING_TABLET, discard the event and poll the next one.
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getFilePath());
return null;
case EMPTY:
case USING_TSFILE:
@@ -539,6 +441,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
LOGGER.error(errorMessage);
PipeDataNodeAgent.runtime()
.report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getFilePath());
return null;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
index 4b300355c80..7549042e676 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
@@ -82,10 +82,7 @@ public class PipeRealtimeDataRegionLogExtractor extends PipeRealtimeDataRegionEx
private void extractTsFileInsertion(PipeRealtimeEvent event) {
final PipeTsFileInsertionEvent tsFileInsertionEvent =
(PipeTsFileInsertionEvent) event.getEvent();
- if (!(tsFileInsertionEvent.isLoaded()
- // some insert nodes in the tsfile epoch are not captured by pipe
- || tsFileInsertionEvent.getFileStartTime()
- < event.getTsFileEpoch().getInsertNodeMinTime())) {
+ if (!(tsFileInsertionEvent.isLoaded())) {
// All data in the tsfile epoch has been extracted in tablet mode, so we should
// simply ignore this event.
event.decreaseReferenceCount(PipeRealtimeDataRegionLogExtractor.class.getName(), false);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
index 8072499b3da..f189af5726d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch.TsFileEpoch;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
@@ -50,6 +51,8 @@ public class PipeRealtimeDataRegionTsFileExtractor extends PipeRealtimeDataRegio
}
event.getTsFileEpoch().migrateState(this, state -> TsFileEpoch.State.USING_TSFILE);
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .registerProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getResource());
if (!(event.getEvent() instanceof TsFileInsertionEvent)) {
event.decreaseReferenceCount(PipeRealtimeDataRegionTsFileExtractor.class.getName(), false);
@@ -111,12 +114,16 @@ public class PipeRealtimeDataRegionTsFileExtractor extends PipeRealtimeDataRegio
LOGGER.error(errorMessage);
PipeDataNodeAgent.runtime()
.report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(
+ dataRegionId, pipeName, realtimeEvent.getTsFileEpoch().getFilePath());
}
realtimeEvent.decreaseReferenceCount(
PipeRealtimeDataRegionTsFileExtractor.class.getName(), false);
if (suppliedEvent != null) {
+ maySkipIndex4Event(realtimeEvent);
return suppliedEvent;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
index ed91f636ac1..8e415772438 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
@@ -26,7 +26,6 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
import org.apache.iotdb.commons.pipe.metric.PipeEventCounter;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
-import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEventFactory;
@@ -35,7 +34,6 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.matcher.CachedSche
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.matcher.PipeDataRegionMatcher;
import org.apache.iotdb.db.pipe.metric.source.PipeAssignerMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
-import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,8 +58,6 @@ public class PipeDataRegionAssigner implements Closeable {
private final String dataRegionId;
- private int counter = 0;
-
private final AtomicReference<ProgressIndex> maxProgressIndexForRealtimeEvent =
new AtomicReference<>(MinimumProgressIndex.INSTANCE);
@@ -107,10 +103,11 @@ public class PipeDataRegionAssigner implements Closeable {
realtimeEvent.decreaseReferenceCount(PipeDataRegionAssigner.class.getName(), false);
final EnrichedEvent innerEvent = realtimeEvent.getEvent();
- eventCounter.decreaseEventCount(innerEvent);
if (innerEvent instanceof PipeHeartbeatEvent) {
((PipeHeartbeatEvent) innerEvent).onAssigned();
}
+
+ eventCounter.decreaseEventCount(innerEvent);
}
private void assignToExtractor(
@@ -128,16 +125,6 @@ public class PipeDataRegionAssigner implements Closeable {
}
if (event.getEvent().isGeneratedByPipe() && !extractor.isForwardingPipeRequests()) {
- // The frequency of progress reports is limited by the counter, while progress
- // reports to TsFileInsertionEvent are not limited.
- if (!(event.getEvent() instanceof TsFileInsertionEvent)) {
- if (counter < PIPE_CONFIG.getPipeNonForwardingEventsProgressReportInterval()) {
- counter++;
- return;
- }
- counter = 0;
- }
-
final ProgressReportEvent reportEvent =
new ProgressReportEvent(
extractor.getPipeName(),
@@ -173,11 +160,6 @@ public class PipeDataRegionAssigner implements Closeable {
extractor.isShouldTransferModFile());
}
- if (innerEvent instanceof PipeTsFileInsertionEvent
- || innerEvent instanceof PipeInsertNodeTabletInsertionEvent) {
- bindOrUpdateProgressIndexForRealtimeEvent(copiedEvent);
- }
-
if (!copiedEvent.increaseReferenceCount(PipeDataRegionAssigner.class.getName())) {
LOGGER.warn(
"The reference count of the event {} cannot be increased, skipping it.",
@@ -188,34 +170,6 @@ public class PipeDataRegionAssigner implements Closeable {
});
}
- private void bindOrUpdateProgressIndexForRealtimeEvent(final PipeRealtimeEvent event) {
- if (PipeTsFileEpochProgressIndexKeeper.getInstance()
- .isProgressIndexAfterOrEquals(
- dataRegionId,
- event.getTsFileEpoch().getFilePath(),
- getProgressIndex4RealtimeEvent(event))) {
- event.bindProgressIndex(maxProgressIndexForRealtimeEvent.get());
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(
- "Data region {} bind {} to event {} because it was flushed prematurely.",
- dataRegionId,
- maxProgressIndexForRealtimeEvent,
- event.coreReportMessage());
- }
- } else {
- maxProgressIndexForRealtimeEvent.updateAndGet(
- index ->
- index.updateToMinimumEqualOrIsAfterProgressIndex(
- getProgressIndex4RealtimeEvent(event)));
- }
- }
-
- private ProgressIndex getProgressIndex4RealtimeEvent(final PipeRealtimeEvent event) {
- return event.getEvent() instanceof PipeTsFileInsertionEvent
- ? ((PipeTsFileInsertionEvent) event.getEvent()).forceGetProgressIndex()
- : event.getProgressIndex();
- }
-
public void startAssignTo(final PipeRealtimeDataRegionExtractor extractor) {
matcher.register(extractor);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java
index da2cde90667..50b7d778fc2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java
@@ -20,6 +20,9 @@
package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import javax.annotation.Nonnull;
import java.util.Map;
import java.util.Map.Entry;
@@ -28,39 +31,40 @@ import java.util.concurrent.ConcurrentHashMap;
public class PipeTsFileEpochProgressIndexKeeper {
- // data region id -> (tsFile path, max progress index)
- private final Map<String, Map<String, ProgressIndex>> progressIndexKeeper =
+ // data region id -> pipeName -> tsFile path -> max progress index
+ private final Map<String, Map<String, Map<String, TsFileResource>>> progressIndexKeeper =
new ConcurrentHashMap<>();
- public synchronized void updateProgressIndex(
- final String dataRegionId, final String tsFileName, final ProgressIndex progressIndex) {
+ public synchronized void registerProgressIndex(
+ final String dataRegionId, final String pipeName, final TsFileResource resource) {
progressIndexKeeper
.computeIfAbsent(dataRegionId, k -> new ConcurrentHashMap<>())
- .compute(
- tsFileName,
- (k, v) ->
- v == null
- ? progressIndex
- : v.updateToMinimumEqualOrIsAfterProgressIndex(progressIndex));
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
+ .putIfAbsent(resource.getTsFilePath(), resource);
}
public synchronized void eliminateProgressIndex(
- final String dataRegionId, final String filePath) {
+ final String dataRegionId, final @Nonnull String pipeName, final String filePath) {
progressIndexKeeper
.computeIfAbsent(dataRegionId, k -> new ConcurrentHashMap<>())
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
.remove(filePath);
}
public synchronized boolean isProgressIndexAfterOrEquals(
- final String dataRegionId, final String tsFilePath, final ProgressIndex progressIndex) {
+ final String dataRegionId,
+ final String pipeName,
+ final String tsFilePath,
+ final ProgressIndex progressIndex) {
return progressIndexKeeper
.computeIfAbsent(dataRegionId, k -> new ConcurrentHashMap<>())
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
.entrySet()
.stream()
.filter(entry -> !Objects.equals(entry.getKey(), tsFilePath))
.map(Entry::getValue)
.filter(Objects::nonNull)
- .anyMatch(index -> !index.isAfter(progressIndex));
+ .anyMatch(resource -> !resource.getMaxProgressIndex().isAfter(progressIndex));
}
//////////////////////////// singleton ////////////////////////////
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java
index c2db4c77c86..83b69ea60ec 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionExtractor;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -29,13 +30,13 @@ import java.util.concurrent.atomic.AtomicReference;
public class TsFileEpoch {
- private final String filePath;
+ private final TsFileResource resource;
private final ConcurrentMap<PipeRealtimeDataRegionExtractor, AtomicReference<State>>
dataRegionExtractor2State;
private final AtomicLong insertNodeMinTime;
- public TsFileEpoch(final String filePath) {
- this.filePath = filePath;
+ public TsFileEpoch(final TsFileResource resource) {
+ this.resource = resource;
this.dataRegionExtractor2State = new ConcurrentHashMap<>();
this.insertNodeMinTime = new AtomicLong(Long.MAX_VALUE);
}
@@ -64,19 +65,19 @@ public class TsFileEpoch {
insertNodeMinTime.updateAndGet(recordedMinTime -> Math.min(recordedMinTime, newComingMinTime));
}
- public long getInsertNodeMinTime() {
- return insertNodeMinTime.get();
+ public TsFileResource getResource() {
+ return resource;
}
public String getFilePath() {
- return filePath;
+ return resource.getTsFilePath();
}
@Override
public String toString() {
return "TsFileEpoch{"
- + "filePath='"
- + filePath
+ + "resource='"
+ + resource
+ '\''
+ ", dataRegionExtractor2State="
+ dataRegionExtractor2State
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java
index 6ee0a738452..9e18258dd17 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java
@@ -56,7 +56,7 @@ public class TsFileEpochManager {
filePath,
path -> {
LOGGER.info("TsFileEpoch not found for TsFile {}, creating a new one", path);
- return new TsFileEpoch(path);
+ return new TsFileEpoch(resource);
});
final TsFileEpoch epoch = filePath2Epoch.remove(filePath);
@@ -79,7 +79,7 @@ public class TsFileEpochManager {
public PipeRealtimeEvent bindPipeInsertNodeTabletInsertionEvent(
PipeInsertNodeTabletInsertionEvent event, InsertNode node, TsFileResource resource) {
final TsFileEpoch epoch =
- filePath2Epoch.computeIfAbsent(resource.getTsFilePath(), TsFileEpoch::new);
+ filePath2Epoch.computeIfAbsent(resource.getTsFilePath(), k -> new TsFileEpoch(resource));
epoch.updateInsertNodeMinTime(node.getMinTime());
return new PipeRealtimeEvent(
event,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
index 452c7188dec..86ac909b0de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeDataR
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -95,11 +94,7 @@ public class PipeInsertionDataNodeListener {
//////////////////////////// listen to events ////////////////////////////
public void listenToTsFile(
- final String dataRegionId,
- final TsFileResource tsFileResource,
- final boolean isLoaded,
- final boolean isGeneratedByPipe) {
- tsFileResource.setGeneratedByPipe(isGeneratedByPipe);
+ final String dataRegionId, final TsFileResource tsFileResource, final boolean isLoaded) {
// We don't judge whether listenToTsFileExtractorCount.get() == 0 here on purpose
// because extractors may use tsfile events when some exceptions occur in the
// insert nodes listening process.
@@ -116,10 +111,7 @@ public class PipeInsertionDataNodeListener {
}
public void listenToInsertNode(
- String dataRegionId,
- WALEntryHandler walEntryHandler,
- InsertNode insertNode,
- TsFileResource tsFileResource) {
+ String dataRegionId, InsertNode insertNode, TsFileResource tsFileResource) {
if (listenToInsertNodeExtractorCount.get() == 0) {
return;
}
@@ -132,7 +124,7 @@ public class PipeInsertionDataNodeListener {
}
assigner.publishToAssign(
- PipeRealtimeEventFactory.createRealtimeEvent(walEntryHandler, insertNode, tsFileResource));
+ PipeRealtimeEventFactory.createRealtimeEvent(insertNode, tsFileResource));
}
public void listenToHeartbeat(boolean shouldPrintMessage) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
index dd3873feaf4..d7f1577a85c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeHeartbeatEventMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
-import org.apache.iotdb.db.pipe.metric.overview.PipeWALInsertNodeCacheMetrics;
import org.apache.iotdb.db.pipe.metric.processor.PipeProcessorMetrics;
import org.apache.iotdb.db.pipe.metric.receiver.PipeDataNodeReceiverMetrics;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionConnectorMetrics;
@@ -47,7 +46,6 @@ public class PipeDataNodeMetrics implements IMetricSet {
PipeProcessorMetrics.getInstance().bindTo(metricService);
PipeDataRegionConnectorMetrics.getInstance().bindTo(metricService);
PipeHeartbeatEventMetrics.getInstance().bindTo(metricService);
- PipeWALInsertNodeCacheMetrics.getInstance().bindTo(metricService);
PipeResourceMetrics.getInstance().bindTo(metricService);
PipeEventCommitMetrics.getInstance().bindTo(metricService);
PipeSchemaRegionListenerMetrics.getInstance().bindTo(metricService);
@@ -65,7 +63,6 @@ public class PipeDataNodeMetrics implements IMetricSet {
PipeProcessorMetrics.getInstance().unbindFrom(metricService);
PipeDataRegionConnectorMetrics.getInstance().unbindFrom(metricService);
PipeHeartbeatEventMetrics.getInstance().unbindFrom(metricService);
- PipeWALInsertNodeCacheMetrics.getInstance().unbindFrom(metricService);
PipeResourceMetrics.getInstance().unbindFrom(metricService);
PipeEventCommitMetrics.getInstance().unbindFrom(metricService);
PipeSchemaRegionListenerMetrics.getInstance().unbindFrom(metricService);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
index 73d58285345..0308e9b5b63 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
@@ -59,8 +59,6 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
private Timer insertNodeTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer tsfileTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- private final InsertNodeEMA insertNodeEventCountEMA = new InsertNodeEMA();
-
private double lastDataRegionCommitSmoothingValue = Long.MAX_VALUE;
private double lastSchemaRegionCommitSmoothingValue = Long.MAX_VALUE;
@@ -102,11 +100,6 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
heartbeatEventCount.decrementAndGet();
}
- double getRemainingInsertEventSmoothingCount() {
- insertNodeEventCountEMA.update(insertNodeEventCount.get());
- return insertNodeEventCountEMA.insertNodeEMAValue;
- }
-
public long getRemainingNonHeartbeatEvents() {
final long remainingEvents =
tsfileEventCount.get()
@@ -147,7 +140,7 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
*
* @return The estimated remaining time
*/
- double getRemainingTime() {
+ public double getRemainingTime() {
final PipeRateAverage pipeRemainingTimeCommitRateAverageTime =
PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime();
@@ -282,17 +275,4 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
dataRegionCommitMeter.set(null);
schemaRegionCommitMeter.set(null);
}
-
- private static class InsertNodeEMA {
- private double insertNodeEMAValue;
-
- public void update(final double newValue) {
- final double alpha = PipeConfig.getInstance().getPipeRemainingInsertNodeCountEMAAlpha();
- if (insertNodeEMAValue == 0) {
- insertNodeEMAValue = newValue;
- } else {
- insertNodeEMAValue = alpha * newValue + (1 - alpha) * insertNodeEMAValue;
- }
- }
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
index 677d758a162..1840093a347 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
@@ -20,11 +20,12 @@
package org.apache.iotdb.db.pipe.metric.overview;
import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
@@ -54,7 +55,7 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
private static Histogram PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
- private static Histogram PIPE_DATANODE_EVENT_TRANSFER_TIME_HISTOGRAM =
+ private static Histogram PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
//////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
@@ -68,7 +69,7 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
MetricLevel.IMPORTANT,
Tag.NAME.toString(),
"insert_node");
- PIPE_DATANODE_EVENT_TRANSFER_TIME_HISTOGRAM =
+ PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM =
metricService.getOrCreateHistogram(
Metric.PIPE_DATANODE_EVENT_TRANSFER.toString(),
MetricLevel.IMPORTANT,
@@ -103,6 +104,35 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
Tag.CREATION_TIME.toString(),
String.valueOf(operator.getCreationTime()));
+ // Resources
+ metricService.createAutoGauge(
+ Metric.PIPE_FLOATING_MEMORY_USAGE.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeAgent.task(),
+ a -> a.getFloatingMemoryUsageInByte(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.createAutoGauge(
+ Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeResourceManager.tsfile(),
+ a -> a.getLinkedTsFileCount(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.createAutoGauge(
+ Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeResourceManager.tsfile(),
+ a -> a.getTotalLinkedTsFileSize(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+
operator.setInsertNodeTransferTimer(
metricService.getOrCreateTimer(
Metric.PIPE_INSERT_NODE_EVENT_TRANSFER_TIME.toString(),
@@ -118,31 +148,6 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
operator.getPipeName()));
}
- public boolean mayRemainingInsertEventExceedLimit(final String pipeID) {
- if (Objects.isNull(metricService)) {
- return true;
- }
-
- if (remainingEventAndTimeOperatorMap.values().stream()
- .map(PipeDataNodeRemainingEventAndTimeOperator::getRemainingInsertEventSmoothingCount)
- .reduce(0d, Double::sum)
- > PipeConfig.getInstance().getPipeMaxAllowedTotalRemainingInsertEventCount()) {
- return true;
- }
-
- final PipeDataNodeRemainingEventAndTimeOperator operator =
- remainingEventAndTimeOperatorMap.get(pipeID);
- if (Objects.isNull(operator)) {
- LOGGER.warn(
- "Failed to get remaining insert event, RemainingEventAndTimeOperator({}) does not exist, will degrade anyway",
- pipeID);
- return true;
- }
-
- return operator.getRemainingInsertEventSmoothingCount()
- > PipeConfig.getInstance().getPipeMaxAllowedRemainingInsertEventCountPerPipe();
- }
-
@Override
public void unbindFrom(final AbstractMetricService metricService) {
ImmutableSet.copyOf(remainingEventAndTimeOperatorMap.keySet()).forEach(this::deregister);
@@ -184,6 +189,27 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
operator.getPipeName(),
Tag.CREATION_TIME.toString(),
String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_FLOATING_MEMORY_USAGE.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
metricService.remove(
MetricType.TIMER,
Metric.PIPE_INSERT_NODE_EVENT_TRANSFER_TIME.toString(),
@@ -280,7 +306,7 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
operator.decreaseTsFileEventCount();
operator.getTsFileTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
- PIPE_DATANODE_EVENT_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+ PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
}
public void increaseHeartbeatEventCount(final String pipeName, final long creationTime) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
index c9115575000..37f8eb5e26a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
@@ -22,10 +22,9 @@ package org.apache.iotdb.db.pipe.metric.overview;
import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
import org.apache.iotdb.metrics.utils.MetricLevel;
@@ -34,6 +33,7 @@ import org.apache.iotdb.metrics.utils.MetricType;
public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_USED_MEMORY = "PipeUsedMemory";
+ private static final String PIPE_USED_FLOATING_MEMORY = "PipeUsedFloatingMemory";
private static final String PIPE_TABLET_USED_MEMORY = "PipeTabletUsedMemory";
@@ -41,6 +41,8 @@ public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_TOTAL_MEMORY = "PipeTotalMemory";
+ private static final String PIPE_FLOATING_MEMORY = "PipeFloatingMemory";
+
//////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
@Override
@@ -74,22 +76,20 @@ public class PipeResourceMetrics implements IMetricSet {
o -> PipeMemoryManager.getTotalNonFloatingMemorySizeInBytes(),
Tag.NAME.toString(),
PIPE_TOTAL_MEMORY);
- // resource reference count
metricService.createAutoGauge(
- Metric.PIPE_PINNED_MEMTABLE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.wal(),
- PipeWALResourceManager::getPinnedWalCount);
- metricService.createAutoGauge(
- Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ Metric.PIPE_MEM.toString(),
MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.tsfile(),
- PipeTsFileResourceManager::getLinkedTsfileCount);
+ PipeDataNodeResourceManager.memory(),
+ o -> PipeMemoryManager.getTotalFloatingMemorySizeInBytes(),
+ Tag.NAME.toString(),
+ PIPE_FLOATING_MEMORY);
metricService.createAutoGauge(
- Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ Metric.PIPE_MEM.toString(),
MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.tsfile(),
- PipeTsFileResourceManager::getTotalLinkedTsfileSize);
+ PipeDataNodeResourceManager.memory(),
+ o -> PipeDataNodeAgent.task().getAllFloatingMemoryUsageInByte(),
+ Tag.NAME.toString(),
+ PIPE_USED_FLOATING_MEMORY);
// phantom reference count
metricService.createAutoGauge(
Metric.PIPE_PHANTOM_REFERENCE_COUNT.toString(),
@@ -115,8 +115,17 @@ public class PipeResourceMetrics implements IMetricSet {
PIPE_TS_FILE_USED_MEMORY);
metricService.remove(
MetricType.AUTO_GAUGE, Metric.PIPE_MEM.toString(), Tag.NAME.toString(), PIPE_TOTAL_MEMORY);
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_MEM.toString(),
+ Tag.NAME.toString(),
+ PIPE_FLOATING_MEMORY);
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_MEM.toString(),
+ Tag.NAME.toString(),
+ PIPE_USED_FLOATING_MEMORY);
// resource reference count
- metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_PINNED_MEMTABLE_COUNT.toString());
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_COUNT.toString());
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_SIZE.toString());
// phantom reference count
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
deleted file mode 100644
index b2e605bf84c..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.metric.overview;
-
-import org.apache.iotdb.commons.service.metric.enums.Metric;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
-import org.apache.iotdb.metrics.AbstractMetricService;
-import org.apache.iotdb.metrics.metricsets.IMetricSet;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.MetricType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class PipeWALInsertNodeCacheMetrics implements IMetricSet {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALInsertNodeCacheMetrics.class);
-
- //////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
-
- @Override
- public void bindTo(AbstractMetricService metricService) {
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_RATE.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheHitRate);
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_COUNT.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheHitCount);
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_REQUEST_COUNT.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheRequestCount);
- }
-
- @Override
- public void unbindFrom(AbstractMetricService metricService) {
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_RATE.toString());
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_COUNT.toString());
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_REQUEST_COUNT.toString());
- }
-
- //////////////////////////// singleton ////////////////////////////
-
- private static class PipeWALInsertNodeCacheMetricsHolder {
-
- private static final PipeWALInsertNodeCacheMetrics INSTANCE =
- new PipeWALInsertNodeCacheMetrics();
-
- private PipeWALInsertNodeCacheMetricsHolder() {
- // empty constructor
- }
- }
-
- public static PipeWALInsertNodeCacheMetrics getInstance() {
- return PipeWALInsertNodeCacheMetricsHolder.INSTANCE;
- }
-
- private PipeWALInsertNodeCacheMetrics() {
- // empty constructor
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java
index 80ebb272c96..49ac24f3e4c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java
@@ -23,7 +23,9 @@ import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtask;
import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingHistogram;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Histogram;
import org.apache.iotdb.metrics.type.Rate;
import org.apache.iotdb.metrics.type.Timer;
import org.apache.iotdb.metrics.utils.MetricLevel;
@@ -44,6 +46,14 @@ public class PipeDataRegionConnectorMetrics implements IMetricSet {
private static final Logger LOGGER =
LoggerFactory.getLogger(PipeDataRegionConnectorMetrics.class);
+ public static Histogram tabletBatchSizeHistogram = new DoNothingHistogram();
+
+ public static Histogram tsFileBatchSizeHistogram = new DoNothingHistogram();
+
+ public static Histogram tabletBatchTimeIntervalHistogram = new DoNothingHistogram();
+
+ public static Histogram tsFileBatchTimeIntervalHistogram = new DoNothingHistogram();
+
@SuppressWarnings("java:S3077")
private volatile AbstractMetricService metricService;
@@ -66,6 +76,22 @@ public class PipeDataRegionConnectorMetrics implements IMetricSet {
for (String taskID : taskIDs) {
createMetrics(taskID);
}
+
+ tabletBatchSizeHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_INSERT_NODE_BATCH_SIZE.toString(), MetricLevel.IMPORTANT);
+
+ tsFileBatchSizeHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_TSFILE_BATCH_SIZE.toString(), MetricLevel.IMPORTANT);
+
+ tabletBatchTimeIntervalHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_INSERT_NODE_BATCH_TIME_COST.toString(), MetricLevel.IMPORTANT);
+
+ tsFileBatchTimeIntervalHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_TSFILE_BATCH_TIME_COST.toString(), MetricLevel.IMPORTANT);
}
private void createMetrics(final String taskID) {
@@ -230,6 +256,14 @@ public class PipeDataRegionConnectorMetrics implements IMetricSet {
LOGGER.warn(
"Failed to unbind from pipe data region connector metrics, connector map not empty");
}
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_INSERT_NODE_BATCH_SIZE.toString());
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_TSFILE_BATCH_SIZE.toString());
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_INSERT_NODE_BATCH_TIME_COST.toString());
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_TSFILE_BATCH_TIME_COST.toString());
}
private void removeMetrics(final String taskID) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
index cb1ba0b9ad9..0e1f0f2b033 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
@@ -114,7 +114,6 @@ public class CombineRequest extends TPipeTransferReq {
version = transferReq.version;
type = transferReq.type;
- body = transferReq.body;
return this;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
index b20904a0e2b..752be403008 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
@@ -99,7 +99,6 @@ public class FetchCombineResultRequest extends TPipeTransferReq {
version = transferReq.version;
type = transferReq.type;
- body = transferReq.body;
return this;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index 7719a338fb6..9c841200ec2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -174,6 +174,14 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
case HANDSHAKE_DATANODE_V1:
{
try {
+ if (PipeConfig.getInstance().isPipeEnableMemoryCheck()
+ && PipeDataNodeResourceManager.memory().getFreeMemorySizeInBytes()
+ < PipeConfig.getInstance().getPipeMinimumReceiverMemory()) {
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(),
+ "The receiver memory is not enough to handle the handshake request from datanode."));
+ }
return handleTransferHandshakeV1(
PipeTransferDataNodeHandshakeV1Req.fromTPipeTransferReq(req));
} finally {
@@ -184,6 +192,14 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
case HANDSHAKE_DATANODE_V2:
{
try {
+ if (PipeConfig.getInstance().isPipeEnableMemoryCheck()
+ && PipeDataNodeResourceManager.memory().getFreeMemorySizeInBytes()
+ < PipeConfig.getInstance().getPipeMinimumReceiverMemory()) {
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(),
+ "The receiver memory is not enough to handle the handshake request from datanode."));
+ }
return handleTransferHandshakeV2(
PipeTransferDataNodeHandshakeV2Req.fromTPipeTransferReq(req));
} finally {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
index 573106e45c2..aaf9eff454b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
@@ -19,24 +19,19 @@
package org.apache.iotdb.db.pipe.resource;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.resource.log.PipeLogManager;
import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager;
import org.apache.iotdb.commons.pipe.resource.snapshot.PipeSnapshotResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.pipe.resource.ref.PipeDataNodePhantomReferenceManager;
import org.apache.iotdb.db.pipe.resource.snapshot.PipeDataNodeSnapshotResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeCompactionManager;
import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.hardlink.PipeWALHardlinkResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.selfhost.PipeWALSelfHostResourceManager;
-
-import java.util.concurrent.atomic.AtomicReference;
public class PipeDataNodeResourceManager {
private final PipeTsFileResourceManager pipeTsFileResourceManager;
- private final AtomicReference<PipeWALResourceManager> pipeWALResourceManager;
+ private final PipeCompactionManager pipeCompactionManager;
private final PipeSnapshotResourceManager pipeSnapshotResourceManager;
private final PipeMemoryManager pipeMemoryManager;
private final PipeLogManager pipeLogManager;
@@ -46,18 +41,8 @@ public class PipeDataNodeResourceManager {
return PipeResourceManagerHolder.INSTANCE.pipeTsFileResourceManager;
}
- public static PipeWALResourceManager wal() {
- if (PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get() == null) {
- synchronized (PipeResourceManagerHolder.INSTANCE) {
- if (PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get() == null) {
- PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.set(
- PipeConfig.getInstance().getPipeHardLinkWALEnabled()
- ? new PipeWALHardlinkResourceManager()
- : new PipeWALSelfHostResourceManager());
- }
- }
- }
- return PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get();
+ public static PipeCompactionManager compaction() {
+ return PipeResourceManagerHolder.INSTANCE.pipeCompactionManager;
}
public static PipeSnapshotResourceManager snapshot() {
@@ -80,7 +65,7 @@ public class PipeDataNodeResourceManager {
private PipeDataNodeResourceManager() {
pipeTsFileResourceManager = new PipeTsFileResourceManager();
- pipeWALResourceManager = new AtomicReference<>();
+ pipeCompactionManager = new PipeCompactionManager();
pipeSnapshotResourceManager = new PipeDataNodeSnapshotResourceManager();
pipeMemoryManager = new PipeMemoryManager();
pipeLogManager = new PipeLogManager();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
index 690a7c54083..8a87cf8bad0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
@@ -54,9 +54,6 @@ public class PipeMemoryManager {
private volatile long usedMemorySizeInBytesOfTsFiles;
- private static final double FLOATING_MEMORY_RATIO =
- PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion();
-
// Only non-zero memory blocks will be added to this set.
private final Set<PipeMemoryBlock> allocatedBlocks = new HashSet<>();
@@ -104,18 +101,6 @@ public class PipeMemoryManager {
* getTotalNonFloatingMemorySizeInBytes();
}
- public long getAllocatedMemorySizeInBytesOfWAL() {
- return (long)
- (PipeConfig.getInstance().getPipeDataStructureWalMemoryProportion()
- * getTotalNonFloatingMemorySizeInBytes());
- }
-
- public long getAllocatedMemorySizeInBytesOfBatch() {
- return (long)
- (PipeConfig.getInstance().getPipeDataStructureBatchMemoryProportion()
- * getTotalNonFloatingMemorySizeInBytes());
- }
-
public boolean isEnough4TabletParsing() {
return (double) usedMemorySizeInBytesOfTablets + (double) usedMemorySizeInBytesOfTsFiles
< EXCEED_PROTECT_THRESHOLD * allowedMaxMemorySizeInBytesOfTabletsAndTsFiles()
@@ -648,15 +633,29 @@ public class PipeMemoryManager {
return usedMemorySizeInBytesOfTsFiles;
}
+ public long getAllocatedMemorySizeInBytesOfBatch() {
+ return (long)
+ (PipeConfig.getInstance().getPipeDataStructureBatchMemoryProportion()
+ * getTotalNonFloatingMemorySizeInBytes());
+ }
+
public long getFreeMemorySizeInBytes() {
return TOTAL_MEMORY_SIZE_IN_BYTES - usedMemorySizeInBytes;
}
public static long getTotalNonFloatingMemorySizeInBytes() {
- return (long) (TOTAL_MEMORY_SIZE_IN_BYTES * (1 - FLOATING_MEMORY_RATIO));
+ return (long)
+ (TOTAL_MEMORY_SIZE_IN_BYTES
+ * (1 - PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion()));
}
public static long getTotalFloatingMemorySizeInBytes() {
- return (long) (TOTAL_MEMORY_SIZE_IN_BYTES * FLOATING_MEMORY_RATIO);
+ return (long)
+ (TOTAL_MEMORY_SIZE_IN_BYTES
+ * PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion());
+ }
+
+ public static long getTotalMemorySizeInBytes() {
+ return TOTAL_MEMORY_SIZE_IN_BYTES;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
index cd9a0a79cae..d82c26144d6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
@@ -247,8 +247,8 @@ public class PipeMemoryWeightUtil {
return totalSizeInBytes;
}
- public static long calculateBatchDataRamBytesUsed(BatchData batchData) {
- long totalSizeInBytes = 0;
+ public static int calculateBatchDataRamBytesUsed(BatchData batchData) {
+ int totalSizeInBytes = 0;
// timestamp
totalSizeInBytes += 8;
@@ -263,16 +263,16 @@ public class PipeMemoryWeightUtil {
continue;
}
// consider variable references (plus 8) and memory alignment (round up to 8)
- totalSizeInBytes += roundUpToMultiple(primitiveType.getSize() + 8L, 8);
+ totalSizeInBytes += roundUpToMultiple(primitiveType.getSize() + 8, 8);
}
} else {
if (type.isBinary()) {
final Binary binary = batchData.getBinary();
// refer to org.apache.tsfile.utils.TsPrimitiveType.TsBinary.getSize
totalSizeInBytes +=
- roundUpToMultiple((binary == null ? 8 : binary.ramBytesUsed()) + 8L, 8);
+ roundUpToMultiple((binary == null ? 8 : binary.getLength() + 8) + 8, 8);
} else {
- totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8L, 8);
+ totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8, 8);
}
}
}
@@ -287,7 +287,7 @@ public class PipeMemoryWeightUtil {
* @param n The specified multiple.
* @return The nearest multiple of n greater than or equal to num.
*/
- private static long roundUpToMultiple(long num, int n) {
+ private static int roundUpToMultiple(int num, int n) {
if (n == 0) {
throw new IllegalArgumentException("The multiple n must be greater than 0");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
new file mode 100644
index 00000000000..03860fda8f8
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.resource.tsfile;
+
+import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
+import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtaskLifeCycle;
+import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeRealtimePriorityBlockingQueue;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.pipe.api.event.Event;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+public class PipeCompactionManager {
+
+ private final Set<PipeConnectorSubtaskLifeCycle> pipeConnectorSubtaskLifeCycles =
+ new CopyOnWriteArraySet<>();
+
+ public void registerPipeConnectorSubtaskLifeCycle(
+ final PipeConnectorSubtaskLifeCycle pipeConnectorSubtaskLifeCycle) {
+ pipeConnectorSubtaskLifeCycles.add(pipeConnectorSubtaskLifeCycle);
+ }
+
+ public void deregisterPipeConnectorSubtaskLifeCycle(
+ final PipeConnectorSubtaskLifeCycle pipeConnectorSubtaskLifeCycle) {
+ pipeConnectorSubtaskLifeCycles.remove(pipeConnectorSubtaskLifeCycle);
+ }
+
+ public void emitResult(
+ final String storageGroupName,
+ final String dataRegionId,
+ final long timePartition,
+ final List<TsFileResource> seqFileResources,
+ final List<TsFileResource> unseqFileResources,
+ final List<TsFileResource> targetFileResources) {
+ final Set<TsFileResource> sourceFileResources = new HashSet<>(seqFileResources);
+ sourceFileResources.addAll(unseqFileResources);
+
+ for (final PipeConnectorSubtaskLifeCycle lifeCycle : pipeConnectorSubtaskLifeCycles) {
+ final UnboundedBlockingPendingQueue<Event> pendingQueue = lifeCycle.getPendingQueue();
+ if (pendingQueue instanceof PipeRealtimePriorityBlockingQueue) {
+ final PipeRealtimePriorityBlockingQueue realtimePriorityBlockingQueue =
+ (PipeRealtimePriorityBlockingQueue) pendingQueue;
+ realtimePriorityBlockingQueue.replace(
+ dataRegionId, sourceFileResources, targetFileResources);
+ }
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFilePublicResource.java
similarity index 63%
copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFilePublicResource.java
index bf789b9732d..47134fe117c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFilePublicResource.java
@@ -23,8 +23,6 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -36,106 +34,27 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-public class PipeTsFileResource implements AutoCloseable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResource.class);
-
- public static final long TSFILE_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
+public class PipeTsFilePublicResource extends PipeTsFileResource {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFilePublicResource.class);
public static final float MEMORY_SUFFICIENT_THRESHOLD = 0.7f;
-
- private final File hardlinkOrCopiedFile;
- private final boolean isTsFile;
-
- /** this TsFileResource is used to track the {@link TsFileResourceStatus} of original TsFile. * */
- private final TsFileResource tsFileResource;
-
- private volatile long fileSize = -1L;
-
- private final AtomicInteger referenceCount;
- private final AtomicLong lastUnpinToZeroTime;
private PipeMemoryBlock allocatedMemoryBlock;
private Map<IDeviceID, List<String>> deviceMeasurementsMap = null;
private Map<IDeviceID, Boolean> deviceIsAlignedMap = null;
private Map<String, TSDataType> measurementDataTypeMap = null;
- public PipeTsFileResource(
- final File hardlinkOrCopiedFile,
- final boolean isTsFile,
- final TsFileResource tsFileResource) {
- this.hardlinkOrCopiedFile = hardlinkOrCopiedFile;
- this.isTsFile = isTsFile;
- this.tsFileResource = tsFileResource;
-
- referenceCount = new AtomicInteger(1);
- lastUnpinToZeroTime = new AtomicLong(Long.MAX_VALUE);
- }
-
- public File getFile() {
- return hardlinkOrCopiedFile;
- }
-
- public boolean isOriginalTsFileDeleted() {
- return isTsFile && Objects.nonNull(tsFileResource) && tsFileResource.isDeleted();
- }
-
- public long getFileSize() {
- if (fileSize == -1L) {
- synchronized (this) {
- if (fileSize == -1L) {
- fileSize = hardlinkOrCopiedFile.length();
- }
- }
- }
- return fileSize;
- }
-
- public long getTsFileResourceSize() {
- return Objects.nonNull(tsFileResource) ? tsFileResource.calculateRamSize() : 0;
- }
-
- ///////////////////// Reference Count /////////////////////
-
- public int getReferenceCount() {
- return referenceCount.get();
- }
-
- public int increaseAndGetReference() {
- return referenceCount.addAndGet(1);
- }
-
- public int decreaseAndGetReference() {
- final int finalReferenceCount = referenceCount.addAndGet(-1);
- if (finalReferenceCount == 0) {
- lastUnpinToZeroTime.set(System.currentTimeMillis());
- }
- if (finalReferenceCount < 0) {
- LOGGER.warn("PipeTsFileResource's reference count is decreased to below 0.");
- }
- return finalReferenceCount;
- }
-
- public synchronized boolean closeIfOutOfTimeToLive() {
- if (referenceCount.get() <= 0
- && (deviceMeasurementsMap == null // Not cached yet.
- || System.currentTimeMillis() - lastUnpinToZeroTime.get()
- > TSFILE_MIN_TIME_TO_LIVE_IN_MS)) {
- close();
- return true;
- } else {
- return false;
- }
+ public PipeTsFilePublicResource(File hardlinkOrCopiedFile) {
+ super(hardlinkOrCopiedFile);
}
@Override
- public synchronized void close() {
+ public void close() {
+ super.close();
+
if (deviceMeasurementsMap != null) {
deviceMeasurementsMap = null;
}
@@ -152,53 +71,39 @@ public class PipeTsFileResource implements AutoCloseable {
allocatedMemoryBlock.close();
allocatedMemoryBlock = null;
}
-
- try {
- Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
- } catch (final Exception e) {
- LOGGER.error(
- "PipeTsFileResource: Failed to delete tsfile {} when closing, because {}. Please MANUALLY delete it.",
- hardlinkOrCopiedFile,
- e.getMessage(),
- e);
- }
-
- LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
}
//////////////////////////// Cache Getter ////////////////////////////
- public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap()
+ public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap(final File tsFile)
throws IOException {
- if (deviceMeasurementsMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ if (deviceMeasurementsMap == null) {
+ cacheObjectsIfAbsent(tsFile);
}
return deviceMeasurementsMap;
}
public synchronized Map<IDeviceID, Boolean> tryGetDeviceIsAlignedMap(
- final boolean cacheOtherMetadata) throws IOException {
- if (deviceIsAlignedMap == null && isTsFile) {
+ final boolean cacheOtherMetadata, final File tsFile) throws IOException {
+ if (deviceIsAlignedMap == null) {
if (cacheOtherMetadata) {
- cacheObjectsIfAbsent();
+ cacheObjectsIfAbsent(tsFile);
} else {
- cacheDeviceIsAlignedMapIfAbsent();
+ cacheDeviceIsAlignedMapIfAbsent(tsFile);
}
}
return deviceIsAlignedMap;
}
- public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap() throws IOException {
- if (measurementDataTypeMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap(final File tsFile)
+ throws IOException {
+ if (measurementDataTypeMap == null) {
+ cacheObjectsIfAbsent(tsFile);
}
return measurementDataTypeMap;
}
- synchronized boolean cacheDeviceIsAlignedMapIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
+ synchronized boolean cacheDeviceIsAlignedMapIfAbsent(final File tsFile) throws IOException {
if (allocatedMemoryBlock != null) {
// This means objects are already cached.
@@ -216,13 +121,13 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"Failed to cacheDeviceIsAlignedMapIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
return false;
}
long memoryRequiredInBytes = 0L;
try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, false)) {
+ new TsFileSequenceReader(tsFile.getPath(), true, false)) {
deviceIsAlignedMap = new HashMap<>();
final TsFileDeviceIterator deviceIsAlignedIterator =
sequenceReader.getAllDevicesIteratorWithIsAligned();
@@ -243,22 +148,16 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
deviceIsAlignedMap = null;
return false;
}
- LOGGER.info(
- "PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.",
- hardlinkOrCopiedFile.getPath());
+ LOGGER.info("PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.", tsFile.getPath());
return true;
}
- synchronized boolean cacheObjectsIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
+ synchronized boolean cacheObjectsIfAbsent(final File tsFile) throws IOException {
if (allocatedMemoryBlock != null) {
if (deviceMeasurementsMap != null) {
return true;
@@ -280,13 +179,13 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"Failed to cacheObjectsIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
return false;
}
long memoryRequiredInBytes = 0L;
try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, true)) {
+ new TsFileSequenceReader(tsFile.getPath(), true, true)) {
deviceMeasurementsMap = sequenceReader.getDeviceMeasurementsMap();
memoryRequiredInBytes +=
PipeMemoryWeightUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap);
@@ -316,15 +215,14 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
deviceIsAlignedMap = null;
deviceMeasurementsMap = null;
measurementDataTypeMap = null;
return false;
}
- LOGGER.info(
- "PipeTsFileResource: Cached objects for tsfile {}.", hardlinkOrCopiedFile.getPath());
+ LOGGER.info("PipeTsFileResource: Cached objects for tsfile {}.", tsFile.getPath());
return true;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
index bf789b9732d..8b37f877094 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
@@ -19,73 +19,33 @@
package org.apache.iotdb.db.pipe.resource.tsfile;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
-
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.read.TsFileDeviceIterator;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
import java.nio.file.Files;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
public class PipeTsFileResource implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResource.class);
- public static final long TSFILE_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
- public static final float MEMORY_SUFFICIENT_THRESHOLD = 0.7f;
-
private final File hardlinkOrCopiedFile;
- private final boolean isTsFile;
-
- /** this TsFileResource is used to track the {@link TsFileResourceStatus} of original TsFile. * */
- private final TsFileResource tsFileResource;
private volatile long fileSize = -1L;
private final AtomicInteger referenceCount;
- private final AtomicLong lastUnpinToZeroTime;
- private PipeMemoryBlock allocatedMemoryBlock;
- private Map<IDeviceID, List<String>> deviceMeasurementsMap = null;
- private Map<IDeviceID, Boolean> deviceIsAlignedMap = null;
- private Map<String, TSDataType> measurementDataTypeMap = null;
- public PipeTsFileResource(
- final File hardlinkOrCopiedFile,
- final boolean isTsFile,
- final TsFileResource tsFileResource) {
+ public PipeTsFileResource(final File hardlinkOrCopiedFile) {
this.hardlinkOrCopiedFile = hardlinkOrCopiedFile;
- this.isTsFile = isTsFile;
- this.tsFileResource = tsFileResource;
referenceCount = new AtomicInteger(1);
- lastUnpinToZeroTime = new AtomicLong(Long.MAX_VALUE);
}
public File getFile() {
return hardlinkOrCopiedFile;
}
- public boolean isOriginalTsFileDeleted() {
- return isTsFile && Objects.nonNull(tsFileResource) && tsFileResource.isDeleted();
- }
-
public long getFileSize() {
if (fileSize == -1L) {
synchronized (this) {
@@ -97,64 +57,33 @@ public class PipeTsFileResource implements AutoCloseable {
return fileSize;
}
- public long getTsFileResourceSize() {
- return Objects.nonNull(tsFileResource) ? tsFileResource.calculateRamSize() : 0;
- }
-
///////////////////// Reference Count /////////////////////
public int getReferenceCount() {
return referenceCount.get();
}
- public int increaseAndGetReference() {
- return referenceCount.addAndGet(1);
+ public void increaseReferenceCount() {
+ referenceCount.addAndGet(1);
}
- public int decreaseAndGetReference() {
+ public boolean decreaseReferenceCount() {
final int finalReferenceCount = referenceCount.addAndGet(-1);
if (finalReferenceCount == 0) {
- lastUnpinToZeroTime.set(System.currentTimeMillis());
+ close();
+ return true;
}
if (finalReferenceCount < 0) {
LOGGER.warn("PipeTsFileResource's reference count is decreased to below 0.");
}
- return finalReferenceCount;
- }
-
- public synchronized boolean closeIfOutOfTimeToLive() {
- if (referenceCount.get() <= 0
- && (deviceMeasurementsMap == null // Not cached yet.
- || System.currentTimeMillis() - lastUnpinToZeroTime.get()
- > TSFILE_MIN_TIME_TO_LIVE_IN_MS)) {
- close();
- return true;
- } else {
- return false;
- }
+ return false;
}
@Override
public synchronized void close() {
- if (deviceMeasurementsMap != null) {
- deviceMeasurementsMap = null;
- }
-
- if (deviceIsAlignedMap != null) {
- deviceIsAlignedMap = null;
- }
-
- if (measurementDataTypeMap != null) {
- measurementDataTypeMap = null;
- }
-
- if (allocatedMemoryBlock != null) {
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
- }
-
+ boolean successful = false;
try {
- Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
+ successful = Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
} catch (final Exception e) {
LOGGER.error(
"PipeTsFileResource: Failed to delete tsfile {} when closing, because {}. Please MANUALLY delete it.",
@@ -163,168 +92,8 @@ public class PipeTsFileResource implements AutoCloseable {
e);
}
- LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
- }
-
- //////////////////////////// Cache Getter ////////////////////////////
-
- public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap()
- throws IOException {
- if (deviceMeasurementsMap == null && isTsFile) {
- cacheObjectsIfAbsent();
- }
- return deviceMeasurementsMap;
- }
-
- public synchronized Map<IDeviceID, Boolean> tryGetDeviceIsAlignedMap(
- final boolean cacheOtherMetadata) throws IOException {
- if (deviceIsAlignedMap == null && isTsFile) {
- if (cacheOtherMetadata) {
- cacheObjectsIfAbsent();
- } else {
- cacheDeviceIsAlignedMapIfAbsent();
- }
- }
- return deviceIsAlignedMap;
- }
-
- public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap() throws IOException {
- if (measurementDataTypeMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ if (successful) {
+ LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
}
- return measurementDataTypeMap;
- }
-
- synchronized boolean cacheDeviceIsAlignedMapIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
- if (allocatedMemoryBlock != null) {
- // This means objects are already cached.
- return true;
- }
-
- // See if pipe memory is sufficient to be allocated for TsFileSequenceReader.
- // Only allocate when pipe memory used is less than 50%, because memory here
- // is hard to shrink and may consume too much memory.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(
- PipeConfig.getInstance().getPipeMemoryAllocateForTsFileSequenceReaderInBytes(),
- MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "Failed to cacheDeviceIsAlignedMapIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- return false;
- }
-
- long memoryRequiredInBytes = 0L;
- try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, false)) {
- deviceIsAlignedMap = new HashMap<>();
- final TsFileDeviceIterator deviceIsAlignedIterator =
- sequenceReader.getAllDevicesIteratorWithIsAligned();
- while (deviceIsAlignedIterator.hasNext()) {
- final Pair<IDeviceID, Boolean> deviceIsAlignedPair = deviceIsAlignedIterator.next();
- deviceIsAlignedMap.put(deviceIsAlignedPair.getLeft(), deviceIsAlignedPair.getRight());
- }
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfIDeviceId2Bool(deviceIsAlignedMap);
- }
- // Release memory of TsFileSequenceReader.
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
-
- // Allocate again for the cached objects.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(memoryRequiredInBytes, MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- deviceIsAlignedMap = null;
- return false;
- }
-
- LOGGER.info(
- "PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.",
- hardlinkOrCopiedFile.getPath());
- return true;
- }
-
- synchronized boolean cacheObjectsIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
- if (allocatedMemoryBlock != null) {
- if (deviceMeasurementsMap != null) {
- return true;
- } else {
- // Recalculate it again because only deviceIsAligned map is cached
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
- }
- }
-
- // See if pipe memory is sufficient to be allocated for TsFileSequenceReader.
- // Only allocate when pipe memory used is less than 50%, because memory here
- // is hard to shrink and may consume too much memory.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(
- PipeConfig.getInstance().getPipeMemoryAllocateForTsFileSequenceReaderInBytes(),
- MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "Failed to cacheObjectsIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- return false;
- }
-
- long memoryRequiredInBytes = 0L;
- try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, true)) {
- deviceMeasurementsMap = sequenceReader.getDeviceMeasurementsMap();
- memoryRequiredInBytes +=
- PipeMemoryWeightUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap);
-
- if (Objects.isNull(deviceIsAlignedMap)) {
- deviceIsAlignedMap = new HashMap<>();
- final TsFileDeviceIterator deviceIsAlignedIterator =
- sequenceReader.getAllDevicesIteratorWithIsAligned();
- while (deviceIsAlignedIterator.hasNext()) {
- final Pair<IDeviceID, Boolean> deviceIsAlignedPair = deviceIsAlignedIterator.next();
- deviceIsAlignedMap.put(deviceIsAlignedPair.getLeft(), deviceIsAlignedPair.getRight());
- }
- }
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfIDeviceId2Bool(deviceIsAlignedMap);
-
- measurementDataTypeMap = sequenceReader.getFullPathDataTypeMap();
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfStr2TSDataType(measurementDataTypeMap);
- }
- // Release memory of TsFileSequenceReader.
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
-
- // Allocate again for the cached objects.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(memoryRequiredInBytes, MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- deviceIsAlignedMap = null;
- deviceMeasurementsMap = null;
- measurementDataTypeMap = null;
- return false;
- }
-
- LOGGER.info(
- "PipeTsFileResource: Cached objects for tsfile {}.", hardlinkOrCopiedFile.getPath());
- return true;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
index d00318ae561..bf8b6b86b43 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
@@ -22,8 +22,7 @@ package org.apache.iotdb.db.pipe.resource.tsfile;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -32,88 +31,30 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
import java.io.File;
import java.io.IOException;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
+import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
public class PipeTsFileResourceManager {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResourceManager.class);
- private final Map<String, PipeTsFileResource> hardlinkOrCopiedFileToPipeTsFileResourceMap =
- new ConcurrentHashMap<>();
- private final PipeTsFileResourceSegmentLock segmentLock = new PipeTsFileResourceSegmentLock();
-
- public PipeTsFileResourceManager() {
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "PipeTsFileResourceManager#ttlCheck()",
- this::tryTtlCheck,
- Math.max(PipeTsFileResource.TSFILE_MIN_TIME_TO_LIVE_IN_MS / 1000, 1));
- }
-
- private void tryTtlCheck() {
- try {
- ttlCheck();
- } catch (final InterruptedException e) {
- Thread.currentThread().interrupt();
- LOGGER.warn("failed to try lock when checking TTL because of interruption", e);
- } catch (final Exception e) {
- LOGGER.warn("failed to check TTL of PipeTsFileResource: ", e);
- }
- }
-
- private void ttlCheck() throws InterruptedException {
- final Iterator<Map.Entry<String, PipeTsFileResource>> iterator =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.entrySet().iterator();
- final long timeout =
- PipeConfig.getInstance().getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds() >> 1;
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeTsFileResourceManager.class,
- PipeConfig.getInstance().getPipeTsFilePinMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeTsFilePinMaxLogIntervalRounds(),
- hardlinkOrCopiedFileToPipeTsFileResourceMap.size());
- final StringBuilder logBuilder = new StringBuilder();
- while (iterator.hasNext()) {
- final Map.Entry<String, PipeTsFileResource> entry = iterator.next();
-
- final String hardlinkOrCopiedFile = entry.getKey();
- if (!segmentLock.tryLock(new File(hardlinkOrCopiedFile), timeout, TimeUnit.SECONDS)) {
- LOGGER.warn(
- "failed to try lock when checking TTL for file {} because of timeout ({}s)",
- hardlinkOrCopiedFile,
- timeout);
- continue;
- }
+ // This is used to hold the assigner pinned tsFiles.
+ // Also, it is used to provide metadata cache of the tsFile, and is shared by all the pipe's
+ // tsFiles.
+ private final Map<String, PipeTsFilePublicResource>
+ hardlinkOrCopiedFileToTsFilePublicResourceMap = new ConcurrentHashMap<>();
- try {
- if (entry.getValue().closeIfOutOfTimeToLive()) {
- iterator.remove();
- } else {
- logBuilder.append(
- String.format(
- "<%s , %d times, %d bytes> ",
- entry.getKey(),
- entry.getValue().getReferenceCount(),
- entry.getValue().getFileSize()));
- }
- } catch (final Exception e) {
- LOGGER.warn("failed to close PipeTsFileResource when checking TTL: ", e);
- } finally {
- segmentLock.unlock(new File(hardlinkOrCopiedFile));
- }
- }
- if (logBuilder.length() > 0) {
- logger.ifPresent(l -> l.info("Pipe file {}are still referenced", logBuilder));
- }
- }
+ // PipeName -> TsFilePath -> PipeTsFileResource
+ private final Map<String, Map<String, PipeTsFileResource>>
+ hardlinkOrCopiedFileToPipeTsFileResourceMap = new ConcurrentHashMap<>();
+ private final PipeTsFileResourceSegmentLock segmentLock = new PipeTsFileResourceSegmentLock();
/**
* Given a file, create a hardlink or copy it to pipe dir, maintain a reference count for the
@@ -131,19 +72,16 @@ public class PipeTsFileResourceManager {
* @param file tsfile, resource file or mod file. can be original file or hardlink/copy of
* original file
* @param isTsFile {@code true} to create hardlink, {@code false} to copy file
- * @param tsFileResource the TsFileResource of original TsFile. Ignored if {@param isTsFile} is
- * {@code false}.
* @return the hardlink or copied file
* @throws IOException when create hardlink or copy file failed
*/
public File increaseFileReference(
- final File file, final boolean isTsFile, final TsFileResource tsFileResource)
- throws IOException {
+ final File file, final boolean isTsFile, final @Nullable String pipeName) throws IOException {
// If the file is already a hardlink or copied file,
// just increase reference count and return it
segmentLock.lock(file);
try {
- if (increaseReferenceIfExists(file)) {
+ if (increaseReferenceIfExists(file, pipeName)) {
return file;
}
} finally {
@@ -152,13 +90,11 @@ public class PipeTsFileResourceManager {
// If the file is not a hardlink or copied file, check if there is a related hardlink or
// copied file in pipe dir. if so, increase reference count and return it
- final File hardlinkOrCopiedFile = getHardlinkOrCopiedFileInPipeDir(file);
+ final File hardlinkOrCopiedFile = getHardlinkOrCopiedFileInPipeDir(file, pipeName);
segmentLock.lock(hardlinkOrCopiedFile);
try {
- if (increaseReferenceIfExists(hardlinkOrCopiedFile)) {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap
- .get(hardlinkOrCopiedFile.getPath())
- .getFile();
+ if (increaseReferenceIfExists(hardlinkOrCopiedFile, pipeName)) {
+ return getResourceMap(pipeName).get(hardlinkOrCopiedFile.getPath()).getFile();
}
// If the file is a tsfile, create a hardlink in pipe dir and will return it.
@@ -171,27 +107,57 @@ public class PipeTsFileResourceManager {
// If the file is not a hardlink or copied file, and there is no related hardlink or copied
// file in pipe dir, create a hardlink or copy it to pipe dir, maintain a reference count for
// the hardlink or copied file, and return the hardlink or copied file.
- hardlinkOrCopiedFileToPipeTsFileResourceMap.put(
- resultFile.getPath(), new PipeTsFileResource(resultFile, isTsFile, tsFileResource));
+ if (Objects.nonNull(pipeName)) {
+ hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
+ .put(resultFile.getPath(), new PipeTsFileResource(resultFile));
+ } else {
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.put(
+ resultFile.getPath(), new PipeTsFilePublicResource(resultFile));
+ }
+
+ increasePublicReference(resultFile, pipeName);
+
return resultFile;
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
}
}
- private boolean increaseReferenceIfExists(final File file) {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(file.getPath());
+ private boolean increaseReferenceIfExists(final File file, final @Nullable String pipeName) {
+ final String path = file.getPath();
+ final PipeTsFileResource resource = getResourceMap(pipeName).get(path);
if (resource != null) {
- resource.increaseAndGetReference();
+ resource.increaseReferenceCount();
+ increasePublicReference(file, pipeName);
return true;
}
return false;
}
- public static File getHardlinkOrCopiedFileInPipeDir(final File file) throws IOException {
+ private void increasePublicReference(final File file, final String pipeName) {
+ if (Objects.isNull(pipeName)) {
+ return;
+ }
+ // Increase the assigner's file to avoid hard-link or memory cache cleaning
+ // Note that it does not exist for historical files
+ final String path = getCommonFilePath(file);
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.compute(
+ path,
+ (k, v) -> {
+ if (Objects.isNull(v)) {
+ return new PipeTsFilePublicResource(new File(path));
+ } else {
+ v.increaseReferenceCount();
+ return v;
+ }
+ });
+ }
+
+ public static File getHardlinkOrCopiedFileInPipeDir(
+ final File file, final @Nullable String pipeName) throws IOException {
try {
- return new File(getPipeTsFileDirPath(file), getRelativeFilePath(file));
+ return new File(getPipeTsFileDirPath(file, pipeName), getRelativeFilePath(file));
} catch (final Exception e) {
throw new IOException(
String.format(
@@ -202,22 +168,28 @@ public class PipeTsFileResourceManager {
}
}
- private static String getPipeTsFileDirPath(File file) throws IOException {
+ private static String getPipeTsFileDirPath(File file, final @Nullable String pipeName)
+ throws IOException {
while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
- && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)) {
+ && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
+ && !file.getName().equals(PipeConfig.getInstance().getPipeHardlinkBaseDirName())) {
file = file.getParentFile();
}
return file.getParentFile().getCanonicalPath()
+ File.separator
+ PipeConfig.getInstance().getPipeHardlinkBaseDirName()
+ File.separator
- + PipeConfig.getInstance().getPipeHardlinkTsFileDirName();
+ + PipeConfig.getInstance().getPipeHardlinkTsFileDirName()
+ + (Objects.nonNull(pipeName) ? File.separator + pipeName : "");
}
private static String getRelativeFilePath(File file) {
StringBuilder builder = new StringBuilder(file.getName());
while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
- && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)) {
+ && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
+ && !file.getParentFile()
+ .getName()
+ .equals(PipeConfig.getInstance().getPipeHardlinkTsFileDirName())) {
file = file.getParentFile();
builder =
new StringBuilder(file.getName())
@@ -231,32 +203,68 @@ public class PipeTsFileResourceManager {
* Given a hardlink or copied file, decrease its reference count, if the reference count is 0,
* delete the file. if the given file is not a hardlink or copied file, do nothing.
*
- * @param hardlinkOrCopiedFile the copied or hardlinked file
+ * @param hardlinkOrCopiedFile the copied or hard-linked file
*/
- public void decreaseFileReference(final File hardlinkOrCopiedFile) {
+ public void decreaseFileReference(
+ final File hardlinkOrCopiedFile, final @Nullable String pipeName) {
segmentLock.lock(hardlinkOrCopiedFile);
try {
final String filePath = hardlinkOrCopiedFile.getPath();
- final PipeTsFileResource resource = hardlinkOrCopiedFileToPipeTsFileResourceMap.get(filePath);
- if (resource != null) {
- resource.decreaseAndGetReference();
+ final PipeTsFileResource resource = getResourceMap(pipeName).get(filePath);
+ if (resource != null && resource.decreaseReferenceCount()) {
+ getResourceMap(pipeName).remove(filePath);
}
+ // Decrease the assigner's file to clear hard-link and memory cache
+ // Note that it does not exist for historical files
+ decreasePublicReferenceIfExists(hardlinkOrCopiedFile, pipeName);
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
}
}
+ private void decreasePublicReferenceIfExists(final File file, final @Nullable String pipeName) {
+ if (Objects.isNull(pipeName)) {
+ return;
+ }
+ // Increase the assigner's file to avoid hard-link or memory cache cleaning
+ // Note that it does not exist for historical files
+ final String commonFilePath = getCommonFilePath(file);
+ if (hardlinkOrCopiedFileToTsFilePublicResourceMap.containsKey(commonFilePath)
+ && hardlinkOrCopiedFileToTsFilePublicResourceMap
+ .get(commonFilePath)
+ .decreaseReferenceCount()) {
+ hardlinkOrCopiedFileToPipeTsFileResourceMap.remove(commonFilePath);
+ }
+ }
+
+ // Warning: Shall not be called by the assigner
+ private String getCommonFilePath(final @Nonnull File file) {
+ // If the parent or grandparent is null then this is testing scenario
+ // Skip the "pipeName" of this file
+ return Objects.isNull(file.getParentFile())
+ || Objects.isNull(file.getParentFile().getParentFile())
+ ? file.getPath()
+ : file.getParentFile().getParent() + File.separator + file.getName();
+ }
+
/**
* Get the reference count of the file.
*
* @param hardlinkOrCopiedFile the copied or hardlinked file
* @return the reference count of the file
*/
- public int getFileReferenceCount(final File hardlinkOrCopiedFile) {
+ @TestOnly
+ public int getFileReferenceCount(
+ final File hardlinkOrCopiedFile, final @Nullable String pipeName) {
segmentLock.lock(hardlinkOrCopiedFile);
try {
- final String filePath = hardlinkOrCopiedFile.getPath();
- final PipeTsFileResource resource = hardlinkOrCopiedFileToPipeTsFileResourceMap.get(filePath);
+ final PipeTsFileResource resource =
+ Objects.nonNull(pipeName)
+ ? hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .get(hardlinkOrCopiedFile.getPath())
+ : hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedFile));
return resource != null ? resource.getReferenceCount() : 0;
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
@@ -272,9 +280,14 @@ public class PipeTsFileResourceManager {
public boolean cacheObjectsIfAbsent(final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource != null && resource.cacheObjectsIfAbsent();
+ if (hardlinkOrCopiedTsFile.getParentFile() == null
+ || hardlinkOrCopiedTsFile.getParentFile().getParentFile() == null) {
+ return false;
+ }
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource != null && resource.cacheObjectsIfAbsent(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -284,9 +297,10 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetDeviceMeasurementsMap();
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null ? null : resource.tryGetDeviceMeasurementsMap(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -296,9 +310,12 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile, final boolean cacheOtherMetadata) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetDeviceIsAlignedMap(cacheOtherMetadata);
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null
+ ? null
+ : resource.tryGetDeviceIsAlignedMap(cacheOtherMetadata, hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -308,38 +325,55 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetMeasurementDataTypeMap();
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null
+ ? null
+ : resource.tryGetMeasurementDataTypeMap(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
}
- public void pinTsFileResource(final TsFileResource resource, final boolean withMods)
+ public Map<String, ? extends PipeTsFileResource> getResourceMap(final @Nullable String pipeName) {
+ return Objects.nonNull(pipeName)
+ ? hardlinkOrCopiedFileToPipeTsFileResourceMap.computeIfAbsent(
+ pipeName, k -> new ConcurrentHashMap<>())
+ : hardlinkOrCopiedFileToTsFilePublicResourceMap;
+ }
+
+ public void pinTsFileResource(
+ final TsFileResource resource, final boolean withMods, final @Nullable String pipeName)
throws IOException {
- increaseFileReference(resource.getTsFile(), true, resource);
+ increaseFileReference(resource.getTsFile(), true, pipeName);
if (withMods && resource.getModFile().exists()) {
- increaseFileReference(new File(resource.getModFile().getFilePath()), false, null);
+ increaseFileReference(new File(resource.getModFile().getFilePath()), false, pipeName);
}
}
- public void unpinTsFileResource(final TsFileResource resource) throws IOException {
- final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile());
- decreaseFileReference(pinnedFile);
+ public void unpinTsFileResource(final TsFileResource resource, final @Nullable String pipeName)
+ throws IOException {
+ final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile(), pipeName);
+ decreaseFileReference(pinnedFile, pipeName);
final File modFile = new File(pinnedFile + ModificationFile.FILE_SUFFIX);
if (modFile.exists()) {
- decreaseFileReference(modFile);
+ decreaseFileReference(modFile, pipeName);
}
}
- public int getLinkedTsfileCount() {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.size();
+ public int getLinkedTsFileCount(final @Nonnull String pipeName) {
+ return hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .size();
}
- public long getTotalLinkedTsfileSize() {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.values().stream()
+ public long getTotalLinkedTsFileSize(final @Nonnull String pipeName) {
+ return hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .values()
+ .stream()
.mapToLong(
resource -> {
try {
@@ -351,47 +385,4 @@ public class PipeTsFileResourceManager {
})
.sum();
}
-
- /**
- * Get the total size of linked TsFiles whose original TsFile is deleted (by compaction or else)
- */
- public long getTotalLinkedButDeletedTsfileSize() {
- try {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.values().parallelStream()
- .filter(PipeTsFileResource::isOriginalTsFileDeleted)
- .mapToLong(
- resource -> {
- try {
- return resource.getFileSize();
- } catch (Exception e) {
- LOGGER.warn(
- "failed to get file size of linked but deleted TsFile {}: ", resource, e);
- return 0;
- }
- })
- .sum();
- } catch (final Exception e) {
- LOGGER.warn("failed to get total size of linked but deleted TsFiles: ", e);
- return 0;
- }
- }
-
- public long getTotalLinkedButDeletedTsFileResourceRamSize() {
- long totalLinkedButDeletedTsfileResourceRamSize = 0;
- try {
- for (final Map.Entry<String, PipeTsFileResource> resourceEntry :
- hardlinkOrCopiedFileToPipeTsFileResourceMap.entrySet()) {
- final PipeTsFileResource pipeTsFileResource = resourceEntry.getValue();
- // If the original TsFile is not deleted, the memory of the resource is not counted
- // because the memory of the resource is controlled by TsFileResourceManager.
- if (pipeTsFileResource.isOriginalTsFileDeleted()) {
- totalLinkedButDeletedTsfileResourceRamSize += pipeTsFileResource.getTsFileResourceSize();
- }
- }
- return totalLinkedButDeletedTsfileResourceRamSize;
- } catch (final Exception e) {
- LOGGER.warn("failed to get total size of linked but deleted TsFiles resource ram size: ", e);
- return totalLinkedButDeletedTsfileResourceRamSize;
- }
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
deleted file mode 100644
index 9d1e530a19d..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal;
-
-import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
-import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-public abstract class PipeWALResource implements Closeable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALResource.class);
-
- protected final WALEntryHandler walEntryHandler;
-
- private final AtomicInteger referenceCount;
-
- public static final long WAL_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
- private final AtomicLong lastLogicalPinTime;
- private final AtomicBoolean isPhysicallyPinned;
-
- protected PipeWALResource(WALEntryHandler walEntryHandler) {
- this.walEntryHandler = walEntryHandler;
-
- referenceCount = new AtomicInteger(0);
-
- lastLogicalPinTime = new AtomicLong(0);
- isPhysicallyPinned = new AtomicBoolean(false);
- }
-
- public final void pin() throws PipeRuntimeNonCriticalException {
- if (referenceCount.get() == 0) {
- if (!isPhysicallyPinned.get()) {
- try {
- pinInternal();
- } catch (MemTablePinException e) {
- throw new PipeRuntimeNonCriticalException(
- String.format(
- "failed to pin wal %d, because %s",
- walEntryHandler.getMemTableId(), e.getMessage()));
- }
- isPhysicallyPinned.set(true);
- LOGGER.info("wal {} is pinned by pipe engine", walEntryHandler.getMemTableId());
- } // else means the wal is already pinned, do nothing
-
- // no matter the wal is pinned or not, update the last pin time
- lastLogicalPinTime.set(System.currentTimeMillis());
- }
-
- referenceCount.incrementAndGet();
- }
-
- protected abstract void pinInternal()
- throws MemTablePinException, PipeRuntimeNonCriticalException;
-
- public final void unpin() throws PipeRuntimeNonCriticalException {
- final int finalReferenceCount = referenceCount.get();
-
- if (finalReferenceCount == 1) {
- unpinPhysicallyIfOutOfTimeToLive();
- } else if (finalReferenceCount < 1) {
- throw new PipeRuntimeCriticalException(
- String.format(
- "wal %d is unpinned more than pinned, this should not happen",
- walEntryHandler.getMemTableId()));
- }
-
- referenceCount.decrementAndGet();
- }
-
- protected abstract void unpinInternal()
- throws MemTablePinException, PipeRuntimeNonCriticalException;
-
- /**
- * Invalidate the wal if it is unpinned and out of time to live.
- *
- * @return true if the wal is invalidated, false otherwise
- */
- public final boolean invalidateIfPossible() {
- if (referenceCount.get() > 0) {
- return false;
- }
-
- // referenceCount.get() == 0
- return unpinPhysicallyIfOutOfTimeToLive();
- }
-
- /**
- * Unpin the wal if it is out of time to live.
- *
- * @return true if the wal is unpinned physically (then it can be invalidated), false otherwise
- * @throws PipeRuntimeNonCriticalException if failed to unpin WAL of memtable.
- */
- private boolean unpinPhysicallyIfOutOfTimeToLive() {
- if (isPhysicallyPinned.get()) {
- if (System.currentTimeMillis() - lastLogicalPinTime.get() > WAL_MIN_TIME_TO_LIVE_IN_MS) {
- try {
- unpinInternal();
- } catch (MemTablePinException e) {
- throw new PipeRuntimeNonCriticalException(
- String.format(
- "failed to unpin wal %d, because %s",
- walEntryHandler.getMemTableId(), e.getMessage()));
- }
- isPhysicallyPinned.set(false);
- LOGGER.info(
- "wal {} is unpinned by pipe engine when checking time to live",
- walEntryHandler.getMemTableId());
- return true;
- } else {
- return false;
- }
- } else {
- LOGGER.info(
- "wal {} is not pinned physically when checking time to live",
- walEntryHandler.getMemTableId());
- return true;
- }
- }
-
- @Override
- public final void close() {
- if (isPhysicallyPinned.get()) {
- try {
- unpinInternal();
- } catch (MemTablePinException e) {
- LOGGER.error(
- "failed to unpin wal {} when closing pipe wal resource, because {}",
- walEntryHandler.getMemTableId(),
- e.getMessage());
- }
- isPhysicallyPinned.set(false);
- LOGGER.info(
- "wal {} is unpinned by pipe engine when closing pipe wal resource",
- walEntryHandler.getMemTableId());
- }
-
- referenceCount.set(0);
- }
-
- public int getReferenceCount() {
- return referenceCount.get();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
deleted file mode 100644
index 9c51d79daad..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ConcurrentModificationException;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantLock;
-
-public abstract class PipeWALResourceManager {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALResourceManager.class);
-
- protected final Map<Long, PipeWALResource> memtableIdToPipeWALResourceMap;
-
- private static final int SEGMENT_LOCK_COUNT = 32;
- private final ReentrantLock[] memTableIdSegmentLocks;
-
- protected PipeWALResourceManager() {
- // memTableIdToPipeWALResourceMap can be concurrently accessed by multiple threads
- memtableIdToPipeWALResourceMap = new ConcurrentHashMap<>();
-
- memTableIdSegmentLocks = new ReentrantLock[SEGMENT_LOCK_COUNT];
- for (int i = 0; i < SEGMENT_LOCK_COUNT; i++) {
- memTableIdSegmentLocks[i] = new ReentrantLock();
- }
-
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "PipeWALResourceManager#ttlCheck()",
- this::ttlCheck,
- Math.max(PipeWALResource.WAL_MIN_TIME_TO_LIVE_IN_MS / 1000, 1));
- }
-
- @SuppressWarnings("java:S2222")
- private void ttlCheck() {
- final Iterator<Map.Entry<Long, PipeWALResource>> iterator =
- memtableIdToPipeWALResourceMap.entrySet().iterator();
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeWALResourceManager.class,
- PipeConfig.getInstance().getPipeWalPinMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeWalPinMaxLogIntervalRounds(),
- memtableIdToPipeWALResourceMap.size());
-
- final StringBuilder logBuilder = new StringBuilder();
- try {
- while (iterator.hasNext()) {
- final Map.Entry<Long, PipeWALResource> entry = iterator.next();
- final ReentrantLock lock =
- memTableIdSegmentLocks[(int) (entry.getKey() % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- if (entry.getValue().invalidateIfPossible()) {
- iterator.remove();
- } else {
- logBuilder.append(
- String.format(
- "<%d , %d times> ", entry.getKey(), entry.getValue().getReferenceCount()));
- }
- } finally {
- lock.unlock();
- }
- }
- } catch (final ConcurrentModificationException e) {
- LOGGER.error(
- "Concurrent modification issues happened, skipping the WAL in this round of ttl check",
- e);
- } finally {
- if (logBuilder.length() > 0) {
- logger.ifPresent(l -> l.info("WAL {}are still referenced", logBuilder));
- }
- }
- }
-
- public final void pin(final WALEntryHandler walEntryHandler) throws IOException {
- final long memTableId = walEntryHandler.getMemTableId();
- final ReentrantLock lock = memTableIdSegmentLocks[(int) (memTableId % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- pinInternal(memTableId, walEntryHandler);
- } finally {
- lock.unlock();
- }
- }
-
- protected abstract void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler)
- throws IOException;
-
- public final void unpin(final WALEntryHandler walEntryHandler) throws IOException {
- final long memTableId = walEntryHandler.getMemTableId();
- final ReentrantLock lock = memTableIdSegmentLocks[(int) (memTableId % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- unpinInternal(memTableId, walEntryHandler);
- } finally {
- lock.unlock();
- }
- }
-
- protected abstract void unpinInternal(
- final long memTableId, final WALEntryHandler walEntryHandler) throws IOException;
-
- public int getPinnedWalCount() {
- return Objects.nonNull(memtableIdToPipeWALResourceMap)
- ? memtableIdToPipeWALResourceMap.size()
- : 0;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
deleted file mode 100644
index f1ad513ccc4..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.hardlink;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALHardlinkResource extends PipeWALResource {
-
- private final PipeWALHardlinkResourceManager resourceManager;
-
- protected PipeWALHardlinkResource(
- WALEntryHandler walEntryHandler, PipeWALHardlinkResourceManager resourceManager) {
- super(walEntryHandler);
- this.resourceManager = resourceManager;
- }
-
- @Override
- protected void pinInternal() throws MemTablePinException {
- // TODO: hardlink
- walEntryHandler.pinMemTable();
- }
-
- @Override
- protected void unpinInternal() throws MemTablePinException {
- // TODO: hardlink
- walEntryHandler.unpinMemTable();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
deleted file mode 100644
index eebf766dc36..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.hardlink;
-
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.HashMap;
-import java.util.Map;
-
-public class PipeWALHardlinkResourceManager extends PipeWALResourceManager {
-
- @Override
- protected void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap
- .computeIfAbsent(memTableId, id -> new PipeWALHardlinkResource(walEntryHandler, this))
- .pin();
- }
-
- @Override
- protected void unpinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap.get(memTableId).unpin();
- }
-
- //////////////////////////// hardlink related ////////////////////////////
-
- private final Map<String, Integer> hardlinkToReferenceMap = new HashMap<>();
-
- /**
- * given a file, create a hardlink, maintain a reference count for the hardlink, and return the
- * hardlink.
- *
- * <p>if the given file is already a hardlink, increase its reference count and return it.
- *
- * <p>if the given file is a wal, create a hardlink in pipe dir, increase the reference count of
- * the hardlink and return it.
- *
- * @param file wal file. can be original file or the hardlink of original file
- * @return the hardlink
- * @throws IOException when create hardlink failed
- */
- public synchronized File increaseFileReference(final File file) throws IOException {
- // if the file is already a hardlink, just increase reference count and return it
- if (increaseReferenceIfExists(file.getPath())) {
- return file;
- }
-
- // if the file is not a hardlink, check if there is a related hardlink in pipe dir. if so,
- // increase reference count and return it.
- final File hardlink = getHardlinkInPipeWALDir(file);
- if (increaseReferenceIfExists(hardlink.getPath())) {
- return hardlink;
- }
-
- // if the file is a wal, and there is no related hardlink in pipe dir, create a hardlink to pipe
- // dir, maintain a reference count for the hardlink, and return the hardlink.
- hardlinkToReferenceMap.put(hardlink.getPath(), 1);
- return FileUtils.createHardLink(file, hardlink);
- }
-
- private boolean increaseReferenceIfExists(final String path) {
- hardlinkToReferenceMap.computeIfPresent(path, (k, v) -> v + 1);
- return hardlinkToReferenceMap.containsKey(path);
- }
-
- // TODO: Check me! Make sure the file is not a hardlink.
- // TODO: IF user specify a wal by config, will the method work?
- private static File getHardlinkInPipeWALDir(final File file) throws IOException {
- try {
- return new File(getPipeWALDirPath(file), getRelativeFilePath(file));
- } catch (final Exception e) {
- throw new IOException(
- String.format(
- "failed to get hardlink in pipe dir " + "for file %s, it is not a wal",
- file.getPath()),
- e);
- }
- }
-
- private static String getPipeWALDirPath(File file) throws IOException {
- while (!file.getName().equals(IoTDBConstant.WAL_FOLDER_NAME)) {
- file = file.getParentFile();
- }
-
- return file.getParentFile().getCanonicalPath()
- + File.separator
- + IoTDBConstant.DATA_FOLDER_NAME
- + File.separator
- + PipeConfig.getInstance().getPipeHardlinkBaseDirName()
- + File.separator
- + PipeConfig.getInstance().getPipeHardlinkWALDirName();
- }
-
- private static String getRelativeFilePath(File file) {
- StringBuilder builder = new StringBuilder(file.getName());
- while (!file.getParentFile().getName().equals(IoTDBConstant.WAL_FOLDER_NAME)) {
- file = file.getParentFile();
- builder =
- new StringBuilder(file.getName())
- .append(IoTDBConstant.FILE_NAME_SEPARATOR)
- .append(builder);
- }
- return builder.toString();
- }
-
- /**
- * given a hardlink, decrease its reference count, if the reference count is 0, delete the file.
- * if the given file is not a hardlink, do nothing.
- *
- * @param hardlink the hardlinked file
- * @throws IOException when delete file failed
- */
- public synchronized void decreaseFileReference(final File hardlink) throws IOException {
- final Integer updatedReference =
- hardlinkToReferenceMap.computeIfPresent(
- hardlink.getPath(), (file, reference) -> reference - 1);
-
- if (updatedReference != null && updatedReference == 0) {
- Files.deleteIfExists(hardlink.toPath());
- hardlinkToReferenceMap.remove(hardlink.getPath());
- }
- }
-
- @TestOnly
- public synchronized int getFileReferenceCount(final File hardlink) {
- return hardlinkToReferenceMap.getOrDefault(hardlink.getPath(), 0);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
deleted file mode 100644
index e8e03e64a16..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.selfhost;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALSelfHostResource extends PipeWALResource {
-
- public PipeWALSelfHostResource(WALEntryHandler walEntryHandler) {
- super(walEntryHandler);
- }
-
- @Override
- protected void pinInternal() throws MemTablePinException {
- walEntryHandler.pinMemTable();
- }
-
- @Override
- protected void unpinInternal() throws MemTablePinException {
- walEntryHandler.unpinMemTable();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
deleted file mode 100644
index c7fe0accda2..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.selfhost;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALSelfHostResourceManager extends PipeWALResourceManager {
-
- @Override
- protected void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap
- .computeIfAbsent(memTableId, id -> new PipeWALSelfHostResource(walEntryHandler))
- .pin();
- }
-
- @Override
- protected void unpinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap.get(memTableId).unpin();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index ec1c3686f24..5d7fa891530 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -41,6 +41,9 @@ import org.apache.iotdb.common.rpc.thrift.TTestConnectionResult;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.request.AsyncRequestContext;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.ConfigurationFileUtils;
@@ -272,9 +275,17 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -316,6 +327,18 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig();
+ private final ExecutorService schemaExecutor =
+ new WrappedThreadPoolExecutor(
+ 0,
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount(),
+ 0L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount()),
+ new IoTThreadFactory(ThreadName.SCHEMA_PARALLEL_POOL.getName()),
+ ThreadName.SCHEMA_PARALLEL_POOL.getName(),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+
private static final String SYSTEM = "system";
public DataNodeInternalRPCServiceImpl() {
@@ -1071,6 +1094,10 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
.map(PipeMeta::deserialize4TaskAgent)
.collect(Collectors.toList()));
+ if (Objects.isNull(exceptionMessages)) {
+ return new TPushPipeMetaResp()
+ .setStatus(new TSStatus(TSStatusCode.PIPE_PUSH_META_TIMEOUT.getStatusCode()));
+ }
return exceptionMessages.isEmpty()
? new TPushPipeMetaResp()
.setStatus(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))
@@ -1347,16 +1374,31 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
final List<TSStatus> statusList = Collections.synchronizedList(new ArrayList<>());
final AtomicBoolean hasFailure = new AtomicBoolean(false);
- consensusGroupIdList.parallelStream()
- .forEach(
- consensusGroupId -> {
- final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
- && status.getCode() != TSStatusCode.ONLY_LOGICAL_VIEW.getStatusCode()) {
- hasFailure.set(true);
- }
- statusList.add(status);
- });
+ final Set<Future<?>> schemaFuture = new HashSet<>();
+
+ consensusGroupIdList.forEach(
+ consensusGroupId ->
+ schemaFuture.add(
+ schemaExecutor.submit(
+ () -> {
+ final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && status.getCode() != TSStatusCode.ONLY_LOGICAL_VIEW.getStatusCode()) {
+ hasFailure.set(true);
+ }
+ statusList.add(status);
+ })));
+
+ for (final Future<?> future : schemaFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing internal schema task: ", e);
+ statusList.add(
+ new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ .setMessage(e.toString()));
+ }
+ }
if (hasFailure.get()) {
return RpcUtils.getStatus(statusList);
@@ -1375,15 +1417,30 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
final List<TSStatus> statusList = Collections.synchronizedList(new ArrayList<>());
final AtomicBoolean hasFailure = new AtomicBoolean(false);
- consensusGroupIdList.parallelStream()
- .forEach(
- consensusGroupId -> {
- final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- hasFailure.set(true);
- }
- statusList.add(status);
- });
+ final Set<Future<?>> schemaFuture = new HashSet<>();
+
+ consensusGroupIdList.forEach(
+ consensusGroupId ->
+ schemaFuture.add(
+ schemaExecutor.submit(
+ () -> {
+ final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ hasFailure.set(true);
+ }
+ statusList.add(status);
+ })));
+
+ for (final Future<?> future : schemaFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing internal schema task: ", e);
+ statusList.add(
+ new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ .setMessage(e.toString()));
+ }
+ }
if (hasFailure.get()) {
return RpcUtils.getStatus(statusList);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
index 19d5cd636d4..d34c0c1f438 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
@@ -405,7 +405,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
final long startTime = System.nanoTime();
try {
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
- new LoadTsFileDataTypeConverter(isGeneratedByPipe);
+ new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
final TSStatus status =
loadTsFileDataTypeConverter
@@ -530,7 +530,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
- new LoadTsFileDataTypeConverter(isGeneratedByPipe);
+ new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
final TSStatus status =
loadTsFileStatement.isConvertOnTypeMismatch()
? loadTsFileDataTypeConverter.convertForTreeModel(loadTsFileStatement).orElse(null)
@@ -897,13 +897,12 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
// check device schema: is aligned or not
final boolean isAlignedInTsFile = schemaCache.getDeviceIsAligned(device);
final boolean isAlignedInIoTDB = iotdbDeviceSchemaInfo.isAligned();
- if (isAlignedInTsFile != isAlignedInIoTDB) {
- throw new LoadAnalyzeException(
- String.format(
- "Device %s in TsFile is %s, but in IoTDB is %s.",
- device,
- isAlignedInTsFile ? "aligned" : "not aligned",
- isAlignedInIoTDB ? "aligned" : "not aligned"));
+ if (LOGGER.isInfoEnabled() && isAlignedInTsFile != isAlignedInIoTDB) {
+ LOGGER.info(
+ "Device {} in TsFile is {}, but in IoTDB is {}.",
+ device,
+ isAlignedInTsFile ? "aligned" : "not aligned",
+ isAlignedInIoTDB ? "aligned" : "not aligned");
}
// check timeseries schema
@@ -921,15 +920,14 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
// check datatype
- if (!tsFileSchema.getType().equals(iotdbSchema.getType())) {
- throw new LoadAnalyzeTypeMismatchException(
- String.format(
- "Measurement %s%s%s datatype not match, TsFile: %s, IoTDB: %s",
- device,
- TsFileConstant.PATH_SEPARATOR,
- iotdbSchema.getMeasurementId(),
- tsFileSchema.getType(),
- iotdbSchema.getType()));
+ if (LOGGER.isInfoEnabled() && !tsFileSchema.getType().equals(iotdbSchema.getType())) {
+ LOGGER.info(
+ "Measurement {}{}{} datatype not match, TsFile: {}, IoTDB: {}",
+ device,
+ TsFileConstant.PATH_SEPARATOR,
+ iotdbSchema.getMeasurementId(),
+ tsFileSchema.getType(),
+ iotdbSchema.getType());
}
// check encoding
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java
index 11478cf1f77..c7bda59fd85 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java
@@ -54,6 +54,7 @@ public class DataNodeSchemaLockManager {
queryContext
.getAcquiredLocks()
.forEach(lockType -> locks[lockType.ordinal()].readLock().unlock());
+ queryContext.getAcquiredLocks().clear();
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
index 43465d7ed74..47e2f477dc9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
@@ -549,7 +549,7 @@ public class LoadTsFileScheduler implements IScheduler {
private void convertFailedTsFilesToTabletsAndRetry() {
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
- new LoadTsFileDataTypeConverter(isGeneratedByPipe);
+ new LoadTsFileDataTypeConverter(queryContext, isGeneratedByPipe);
final Iterator<Integer> iterator = failedTsFileNodeIndexes.listIterator();
while (iterator.hasNext()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index f923fb2f341..6fbf870617a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -38,7 +38,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.ShutdownException;
import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.commons.file.SystemFileFactory;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.schema.ttl.TTLCache;
import org.apache.iotdb.commons.service.IService;
import org.apache.iotdb.commons.service.ServiceType;
@@ -54,7 +53,6 @@ import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
import org.apache.iotdb.db.exception.load.LoadReadOnlyException;
import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler;
@@ -219,12 +217,6 @@ public class StorageEngine implements IService {
LOGGER.info(
"Storage Engine recover cost: {}s.",
(System.currentTimeMillis() - startRecoverTime) / 1000);
-
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "StorageEngine#operateFlush",
- () -> operateFlush(new TFlushReq()),
- PipeConfig.getInstance().getPipeStorageEngineFlushTimeIntervalMs() / 1000);
},
ThreadName.STORAGE_ENGINE_RECOVER_TRIGGER.getName());
recoverEndTrigger.start();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 5e9b25bdda1..555d90e6c7c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1898,7 +1898,7 @@ public class DataRegion implements IDataRegionForQuery {
}
/** close all working tsfile processors */
- private List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
+ public List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
writeLock("asyncCloseAllWorkingTsFileProcessors");
List<Future<?>> futures = new ArrayList<>();
int count = 0;
@@ -3296,8 +3296,7 @@ public class DataRegion implements IDataRegionForQuery {
}
// Listen before the tsFile is added into tsFile manager to avoid it being compacted
- PipeInsertionDataNodeListener.getInstance()
- .listenToTsFile(dataRegionId, tsFileResource, true, isGeneratedByPipe);
+ PipeInsertionDataNodeListener.getInstance().listenToTsFile(dataRegionId, tsFileResource, true);
tsFileManager.add(tsFileResource, false);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index c4fba1fb215..992b5406708 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -227,9 +227,9 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask {
new File(targetTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath());
}
- targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
+ targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndex());
targetFile.deserialize();
- targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
+ targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndex());
}
private boolean recoverTaskInfoFromLogFile() throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index afbea98ec29..d0f79cb992b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -287,8 +287,11 @@ public class CompactionUtils {
List<TsFileResource> seqResources,
List<TsFileResource> unseqResources) {
for (TsFileResource targetResource : targetResources) {
+ // Initial value
+ targetResource.setGeneratedByPipe(true);
+ targetResource.setGeneratedByPipeConsensus(true);
for (TsFileResource unseqResource : unseqResources) {
- targetResource.updateProgressIndex(unseqResource.getMaxProgressIndexAfterClose());
+ targetResource.updateProgressIndex(unseqResource.getMaxProgressIndex());
targetResource.setGeneratedByPipe(
unseqResource.isGeneratedByPipe() && targetResource.isGeneratedByPipe());
targetResource.setGeneratedByPipeConsensus(
@@ -296,7 +299,7 @@ public class CompactionUtils {
&& targetResource.isGeneratedByPipeConsensus());
}
for (TsFileResource seqResource : seqResources) {
- targetResource.updateProgressIndex(seqResource.getMaxProgressIndexAfterClose());
+ targetResource.updateProgressIndex(seqResource.getMaxProgressIndex());
targetResource.setGeneratedByPipe(
seqResource.isGeneratedByPipe() && targetResource.isGeneratedByPipe());
targetResource.setGeneratedByPipeConsensus(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java
index 9b730b7e300..3eac19188e9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java
@@ -36,11 +36,4 @@ public class NotifyFlushMemTable extends AbstractMemTable {
public boolean isSignalMemTable() {
return true;
}
-
- @Override
- public boolean isTotallyGeneratedByPipe() {
- // Even though the `isTotallyGeneratedByPipe` for the corresponding memory table of this
- // `NotifyFlushMemTable` might be true, we still return false to ensure data integrity.
- return false;
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index 6df59ed1220..dcae00700ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -67,7 +67,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -122,8 +121,6 @@ public abstract class AbstractMemTable implements IMemTable {
private static final String METRIC_POINT_IN = Metric.POINTS_IN.toString();
- private final AtomicBoolean isTotallyGeneratedByPipe = new AtomicBoolean(true);
-
protected AbstractMemTable() {
this.database = null;
this.dataRegionId = null;
@@ -990,14 +987,4 @@ public abstract class AbstractMemTable implements IMemTable {
this.database = database;
this.dataRegionId = dataRegionId;
}
-
- @Override
- public void markAsNotGeneratedByPipe() {
- this.isTotallyGeneratedByPipe.set(false);
- }
-
- @Override
- public boolean isTotallyGeneratedByPipe() {
- return this.isTotallyGeneratedByPipe.get();
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
index 9de36b73eae..3a7e9c55092 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
@@ -203,9 +203,5 @@ public interface IMemTable extends WALEntryValue {
void setDatabaseAndDataRegionId(String database, String dataRegionId);
- void markAsNotGeneratedByPipe();
-
- boolean isTotallyGeneratedByPipe();
-
void updateMemtablePointCountMetric(InsertNode insertNode, int pointsInserted);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index c91cb7838e3..451986b2f52 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -110,6 +110,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -187,6 +188,8 @@ public class TsFileProcessor {
/** Total memtable size for mem control. */
private long totalMemTableSize;
+ private final AtomicBoolean isTotallyGeneratedByPipe = new AtomicBoolean(true);
+
private static final String FLUSH_QUERY_WRITE_LOCKED = "{}: {} get flushQueryLock write lock";
private static final String FLUSH_QUERY_WRITE_RELEASE =
"{}: {} get flushQueryLock write lock released";
@@ -317,14 +320,11 @@ public class TsFileProcessor {
PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowNode);
if (!insertRowNode.isGeneratedByPipe()) {
- workMemTable.markAsNotGeneratedByPipe();
+ this.isTotallyGeneratedByPipe.set(false);
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- walFlushListener.getWalEntryHandler(),
- insertRowNode,
- tsFileResource);
+ dataRegionInfo.getDataRegion().getDataRegionId(), insertRowNode, tsFileResource);
int pointInserted;
if (insertRowNode.isAligned()) {
@@ -418,14 +418,11 @@ public class TsFileProcessor {
PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowsNode);
if (!insertRowsNode.isGeneratedByPipe()) {
- workMemTable.markAsNotGeneratedByPipe();
+ this.isTotallyGeneratedByPipe.set(false);
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- walFlushListener.getWalEntryHandler(),
- insertRowsNode,
- tsFileResource);
+ dataRegionInfo.getDataRegion().getDataRegionId(), insertRowsNode, tsFileResource);
int pointInserted = 0;
for (InsertRowNode insertRowNode : insertRowsNode.getInsertRowNodeList()) {
@@ -536,14 +533,11 @@ public class TsFileProcessor {
PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertTabletNode);
if (!insertTabletNode.isGeneratedByPipe()) {
- workMemTable.markAsNotGeneratedByPipe();
+ this.isTotallyGeneratedByPipe.set(false);
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- walFlushListener.getWalEntryHandler(),
- insertTabletNode,
- tsFileResource);
+ dataRegionInfo.getDataRegion().getDataRegionId(), insertTabletNode, tsFileResource);
int pointInserted;
try {
@@ -1146,15 +1140,9 @@ public class TsFileProcessor {
// we have to add the memtable into flushingList first and then set the shouldClose tag.
// see https://issues.apache.org/jira/browse/IOTDB-510
IMemTable tmpMemTable = workMemTable == null ? new NotifyFlushMemTable() : workMemTable;
+ tsFileResource.setGeneratedByPipe(isTotallyGeneratedByPipe.get());
try {
- PipeInsertionDataNodeListener.getInstance()
- .listenToTsFile(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- tsFileResource,
- false,
- tmpMemTable.isTotallyGeneratedByPipe());
-
// When invoke closing TsFile after insert data to memTable, we shouldn't flush until invoke
// flushing memTable in System module.
Future<?> future = addAMemtableIntoFlushingList(tmpMemTable);
@@ -1586,6 +1574,12 @@ public class TsFileProcessor {
logger.debug("Start to end file {}", tsFileResource);
}
writer.endFile();
+
+ // Listen after "endFile" to avoid unnecessary waiting for tsFile close
+ // before resource serialization to avoid missing hardlink after restart
+ PipeInsertionDataNodeListener.getInstance()
+ .listenToTsFile(dataRegionInfo.getDataRegion().getDataRegionId(), tsFileResource, false);
+
tsFileResource.serialize();
FileTimeIndexCacheRecorder.getInstance().logFileTimeIndex(tsFileResource);
if (logger.isDebugEnabled()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
index 734c1449dfa..fb7d3303576 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.dataregion.tsfile;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder;
import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager;
@@ -276,6 +277,18 @@ public class TsFileManager {
} finally {
writeUnlock();
}
+
+ // Currently disable
+ if (false) {
+ PipeDataNodeResourceManager.compaction()
+ .emitResult(
+ storageGroupName,
+ dataRegionId,
+ timePartition,
+ seqFileResources,
+ unseqFileResources,
+ targetFileResources);
+ }
}
public boolean contains(TsFileResource tsFileResource, boolean sequence) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 7761ea5c3b1..c1a67400f4d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -28,7 +28,6 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.load.PartitionViolationException;
-import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils.InsertionCompactionCandidateStatus;
@@ -166,13 +165,13 @@ public class TsFileResource {
*/
private TsFileResource originTsFileResource;
- private ProgressIndex maxProgressIndex;
+ private final AtomicReference<ProgressIndex> maxProgressIndex = new AtomicReference<>();
/** used to prevent circular replication in PipeConsensus */
- private boolean isGeneratedByPipeConsensus = false;
+ private volatile boolean isGeneratedByPipeConsensus = false;
/** used to prevent circular replication in Pipe */
- private boolean isGeneratedByPipe = false;
+ private volatile boolean isGeneratedByPipe = false;
private InsertionCompactionCandidateStatus insertionCompactionCandidateStatus =
InsertionCompactionCandidateStatus.NOT_CHECKED;
@@ -268,9 +267,9 @@ public class TsFileResource {
ReadWriteIOUtils.write((String) null, outputStream);
}
- if (maxProgressIndex != null) {
+ if (maxProgressIndex.get() != null) {
TsFileResourceBlockType.PROGRESS_INDEX.serialize(outputStream);
- maxProgressIndex.serialize(outputStream);
+ maxProgressIndex.get().serialize(outputStream);
} else {
TsFileResourceBlockType.EMPTY_BLOCK.serialize(outputStream);
}
@@ -302,7 +301,7 @@ public class TsFileResource {
TsFileResourceBlockType.deserialize(ReadWriteIOUtils.readByte(inputStream));
switch (blockType) {
case PROGRESS_INDEX:
- maxProgressIndex = ProgressIndexType.deserializeFrom(inputStream);
+ maxProgressIndex.set(ProgressIndexType.deserializeFrom(inputStream));
break;
case PIPE_MARK:
isGeneratedByPipeConsensus = ReadWriteIOUtils.readBoolean(inputStream);
@@ -1198,13 +1197,9 @@ public class TsFileResource {
return;
}
- maxProgressIndex =
- (maxProgressIndex == null
- ? progressIndex
- : maxProgressIndex.updateToMinimumEqualOrIsAfterProgressIndex(progressIndex));
-
- PipeTsFileEpochProgressIndexKeeper.getInstance()
- .updateProgressIndex(getDataRegionId(), getTsFilePath(), maxProgressIndex);
+ if (!maxProgressIndex.compareAndSet(null, progressIndex)) {
+ maxProgressIndex.get().updateToMinimumEqualOrIsAfterProgressIndex(progressIndex);
+ }
}
public void setProgressIndex(ProgressIndex progressIndex) {
@@ -1212,22 +1207,12 @@ public class TsFileResource {
return;
}
- maxProgressIndex = progressIndex;
-
- PipeTsFileEpochProgressIndexKeeper.getInstance()
- .updateProgressIndex(getDataRegionId(), getTsFilePath(), maxProgressIndex);
- }
-
- public ProgressIndex getMaxProgressIndexAfterClose() throws IllegalStateException {
- if (getStatus().equals(TsFileResourceStatus.UNCLOSED)) {
- throw new IllegalStateException(
- "Should not get progress index from a unclosing TsFileResource.");
- }
- return getMaxProgressIndex();
+ maxProgressIndex.set(progressIndex);
}
public ProgressIndex getMaxProgressIndex() {
- return maxProgressIndex == null ? MinimumProgressIndex.INSTANCE : maxProgressIndex;
+ final ProgressIndex index = maxProgressIndex.get();
+ return index == null ? MinimumProgressIndex.INSTANCE : index;
}
public boolean isEmpty() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index edc30604a6e..36c404faa7c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -342,7 +342,6 @@ public class WALBuffer extends AbstractWALBuffer {
info.metaData.add(size, searchIndex, walEntry.getMemTableId());
info.memTableId2WalDiskUsage.compute(
walEntry.getMemTableId(), (k, v) -> v == null ? size : v + size);
- walEntry.getWalFlushListener().getWalEntryHandler().setSize(size);
info.fsyncListeners.add(walEntry.getWalFlushListener());
}
@@ -593,13 +592,8 @@ public class WALBuffer extends AbstractWALBuffer {
// notify all waiting listeners
if (forceSuccess) {
- long position = lastFsyncPosition;
for (WALFlushListener fsyncListener : info.fsyncListeners) {
fsyncListener.succeed();
- if (fsyncListener.getWalEntryHandler() != null) {
- fsyncListener.getWalEntryHandler().setEntryPosition(walFileVersionId, position);
- position += fsyncListener.getWalEntryHandler().getSize();
- }
}
lastFsyncPosition = currentWALFileWriter.originalSize();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
index a8e94e2780a..18304e621b7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
@@ -77,14 +77,14 @@ public abstract class WALEntry implements SerializedSize {
} else {
throw new RuntimeException("Unknown WALEntry type");
}
- walFlushListener = new WALFlushListener(wait, value);
+ walFlushListener = new WALFlushListener(wait);
}
protected WALEntry(WALEntryType type, long memTableId, WALEntryValue value, boolean wait) {
this.type = type;
this.memTableId = memTableId;
this.value = value;
- this.walFlushListener = new WALFlushListener(wait, value);
+ this.walFlushListener = new WALFlushListener(wait);
}
public abstract void serialize(IWALByteBufferView buffer);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
index afa6651dfa4..6f801a78457 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
@@ -25,11 +25,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.ILogWriter;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,9 +175,7 @@ public class CheckpointManager implements AutoCloseable {
return;
}
memTableInfo.setFlushed();
- if (!memTableInfo.isPinned()) {
- memTableId2Info.remove(memTableId);
- }
+ memTableId2Info.remove(memTableId);
Checkpoint checkpoint =
new Checkpoint(
CheckpointType.FLUSH_MEMORY_TABLE, Collections.singletonList(memTableInfo));
@@ -261,71 +257,9 @@ public class CheckpointManager implements AutoCloseable {
// endregion
- // region methods for pipe
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable(long memTableId) throws MemTablePinException {
- infoLock.lock();
- try {
- if (!memTableId2Info.containsKey(memTableId)) {
- throw new MemTablePinException(
- String.format(
- "Fail to pin memTable-%d because this memTable doesn't exist in the wal.",
- memTableId));
- }
- MemTableInfo memTableInfo = memTableId2Info.get(memTableId);
- if (!memTableInfo.isPinned()) {
- WALInsertNodeCache.getInstance().addMemTable(memTableId);
- }
- memTableInfo.pin();
- } finally {
- infoLock.unlock();
- }
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable(long memTableId) throws MemTablePinException {
- infoLock.lock();
- try {
- if (!memTableId2Info.containsKey(memTableId)) {
- throw new MemTablePinException(
- String.format(
- "Fail to unpin memTable-%d because this memTable doesn't exist in the wal.",
- memTableId));
- }
- if (!memTableId2Info.get(memTableId).isPinned()) {
- throw new MemTablePinException(
- String.format(
- "Fail to unpin memTable-%d because this memTable hasn't been pinned.", memTableId));
- }
- MemTableInfo memTableInfo = memTableId2Info.get(memTableId);
- memTableInfo.unpin();
- if (!memTableInfo.isPinned()) {
- WALInsertNodeCache.getInstance().removeMemTable(memTableId);
- if (memTableInfo.isFlushed()) {
- memTableId2Info.remove(memTableId);
- }
- }
- } finally {
- infoLock.unlock();
- }
- }
-
- // endregion
-
- /** Get MemTableInfo of oldest unpinned MemTable, whose first version id is smallest. */
- public MemTableInfo getOldestUnpinnedMemTableInfo() {
+ public MemTableInfo getOldestMemTableInfo() {
// find oldest memTable
return activeOrPinnedMemTables().stream()
- .filter(memTableInfo -> !memTableInfo.isPinned())
.min(Comparator.comparingLong(MemTableInfo::getMemTableId))
.orElse(null);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
index 59c2a3be23b..984006b25bc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
@@ -47,8 +47,7 @@ public class MemTableInfo implements WALEntryValue {
// memTable
private IMemTable memTable;
- // memTable pin count
- private int pinCount;
+
// memTable is flushed or not
private boolean flushed;
// data region id
@@ -116,22 +115,6 @@ public class MemTableInfo implements WALEntryValue {
return memTable;
}
- public void pin() {
- this.pinCount++;
- }
-
- public void unpin() {
- this.pinCount--;
- }
-
- public boolean isPinned() {
- return pinCount > 0;
- }
-
- public int getPinCount() {
- return pinCount;
- }
-
public boolean isFlushed() {
return flushed;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
index b03b27a6994..2f257da9adc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.storageengine.dataregion.wal.io;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryPosition;
import java.io.Closeable;
import java.io.DataInputStream;
@@ -50,18 +49,6 @@ public class WALByteBufReader implements Closeable {
}
}
- public WALByteBufReader(WALEntryPosition walEntryPosition) throws IOException {
- WALInputStream walInputStream = walEntryPosition.openReadFileStream();
- try {
- this.logStream = new DataInputStream(walInputStream);
- this.metaData = walInputStream.getWALMetaData();
- this.sizeIterator = metaData.getBuffersSize().iterator();
- } catch (Exception e) {
- walInputStream.close();
- throw e;
- }
- }
-
/** Like {@link Iterator#hasNext()}. */
public boolean hasNext() {
return sizeIterator.hasNext();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
index 38b69f1162b..dfa7bf6bb9d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
@@ -41,9 +41,9 @@ public class WALFakeNode implements IWALNode {
public WALFakeNode(Status status, Exception cause) {
this.status = status;
- this.successListener = new WALFlushListener(false, null);
+ this.successListener = new WALFlushListener(false);
this.successListener.succeed();
- this.failListener = new WALFlushListener(false, null);
+ this.failListener = new WALFlushListener(false);
this.failListener.fail(cause);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
index 35432a8fd6f..003c74763da 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
@@ -49,7 +49,6 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointType;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.MemTableInfo;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALByteBufReader;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
@@ -57,7 +56,6 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.AbstractR
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.AbstractResultListener.Status;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALFlushListener;
-import org.apache.commons.lang3.StringUtils;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.utils.TsFileUtils;
import org.slf4j.Logger;
@@ -183,7 +181,6 @@ public class WALNode implements IWALNode {
buffer.write(walEntry);
// set handler for pipe
- walEntry.getWalFlushListener().getWalEntryHandler().setWalNode(this, walEntry.getMemTableId());
return walEntry.getWalFlushListener();
}
@@ -228,25 +225,6 @@ public class WALNode implements IWALNode {
// region methods for pipe
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable(long memTableId) throws MemTablePinException {
- checkpointManager.pinMemTable(memTableId);
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable(long memTableId) throws MemTablePinException {
- checkpointManager.unpinMemTable(memTableId);
- }
-
// endregion
// region Task to delete outdated .wal files
@@ -270,8 +248,6 @@ public class WALNode implements IWALNode {
// the effective information ratio
private double effectiveInfoRatio = 1.0d;
- private List<Long> pinnedMemTableIds;
-
private int fileIndexAfterFilterSafelyDeleteIndex = Integer.MAX_VALUE;
private List<Long> successfullyDeleted;
private long deleteFileSize;
@@ -297,7 +273,6 @@ public class WALNode implements IWALNode {
this.sortedWalFilesExcludingLast =
Arrays.copyOfRange(allWalFilesOfOneNode, 0, allWalFilesOfOneNode.length - 1);
this.activeOrPinnedMemTables = checkpointManager.activeOrPinnedMemTables();
- this.pinnedMemTableIds = initPinnedMemTableIds();
this.fileIndexAfterFilterSafelyDeleteIndex = initFileIndexAfterFilterSafelyDeleteIndex();
this.successfullyDeleted = new ArrayList<>();
this.deleteFileSize = 0;
@@ -318,20 +293,6 @@ public class WALNode implements IWALNode {
}
}
- private List<Long> initPinnedMemTableIds() {
- List<MemTableInfo> memTableInfos = checkpointManager.activeOrPinnedMemTables();
- if (memTableInfos.isEmpty()) {
- return new ArrayList<>();
- }
- List<Long> pinnedIds = new ArrayList<>();
- for (MemTableInfo memTableInfo : memTableInfos) {
- if (memTableInfo.isFlushed() && memTableInfo.isPinned()) {
- pinnedIds.add(memTableInfo.getMemTableId());
- }
- }
- return pinnedIds;
- }
-
@Override
public void run() {
// The intent of the loop execution here is to try to get as many memTable flush or snapshot
@@ -365,7 +326,7 @@ public class WALNode implements IWALNode {
private void updateEffectiveInfoRationAndUpdateMetric() {
// calculate effective information ratio
long costOfActiveMemTables = checkpointManager.getTotalCostOfActiveMemTables();
- MemTableInfo oldestUnpinnedMemTableInfo = checkpointManager.getOldestUnpinnedMemTableInfo();
+ MemTableInfo oldestUnpinnedMemTableInfo = checkpointManager.getOldestMemTableInfo();
long avgFileSize =
getFileNum() != 0
? getDiskUsage() / getFileNum()
@@ -389,45 +350,10 @@ public class WALNode implements IWALNode {
}
private void summarizeExecuteResult() {
- if (!pinnedMemTableIds.isEmpty()
- || fileIndexAfterFilterSafelyDeleteIndex < sortedWalFilesExcludingLast.length) {
- if (logger.isDebugEnabled()) {
- StringBuilder summary =
- new StringBuilder(
- String.format(
- "wal node-%s delete outdated files summary:the range is: [%d,%d], delete successful is [%s], safely delete file index is: [%s].The following reasons influenced the result: %s",
- identifier,
- WALFileUtils.parseVersionId(sortedWalFilesExcludingLast[0].getName()),
- WALFileUtils.parseVersionId(
- sortedWalFilesExcludingLast[sortedWalFilesExcludingLast.length - 1]
- .getName()),
- StringUtils.join(successfullyDeleted, ","),
- fileIndexAfterFilterSafelyDeleteIndex,
- System.lineSeparator()));
-
- if (!pinnedMemTableIds.isEmpty()) {
- summary
- .append("- MemTable has been flushed but pinned by PIPE, the MemTableId list is : ")
- .append(StringUtils.join(pinnedMemTableIds, ","))
- .append(".")
- .append(System.lineSeparator());
- }
- if (fileIndexAfterFilterSafelyDeleteIndex < sortedWalFilesExcludingLast.length) {
- summary.append(
- String.format(
- "- The data in the wal file was not consumed by the consensus group,current search index is %d, safely delete index is %d",
- getCurrentSearchIndex(), safelyDeletedSearchIndex));
- }
- String summaryLog = summary.toString();
- logger.debug(summaryLog);
- }
-
- } else {
- logger.debug(
- "Successfully delete {} outdated wal files for wal node-{}",
- successfullyDeleted.size(),
- identifier);
- }
+ logger.debug(
+ "Successfully delete {} outdated wal files for wal node-{}",
+ successfullyDeleted.size(),
+ identifier);
}
/** Delete obsolete wal files while recording which succeeded or failed */
@@ -475,20 +401,10 @@ public class WALNode implements IWALNode {
return false;
}
// find oldest memTable
- MemTableInfo oldestMemTableInfo = checkpointManager.getOldestUnpinnedMemTableInfo();
+ MemTableInfo oldestMemTableInfo = checkpointManager.getOldestMemTableInfo();
if (oldestMemTableInfo == null) {
return false;
}
- if (oldestMemTableInfo.isPinned()) {
- logger.warn(
- "Pipe: Effective information ratio {} of wal node-{} is below wal min effective info ratio {}. But fail to delete memTable-{}'s wal files because they are pinned by the Pipe module. Pin count: {}.",
- effectiveInfoRatio,
- identifier,
- config.getWalMinEffectiveInfoRatio(),
- oldestMemTableInfo.getMemTableId(),
- oldestMemTableInfo.getPinCount());
- return false;
- }
IMemTable oldestMemTable = oldestMemTableInfo.getMemTable();
if (oldestMemTable == null) {
return false;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
deleted file mode 100644
index f5d7406f5a6..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
-
-import org.apache.tsfile.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * This handler is used by the Pipe to find the corresponding {@link InsertNode}. Besides, it can
- * try to pin/unpin the {@link WALEntry}s by the memTable id.
- */
-public class WALEntryHandler {
-
- private static final Logger logger = LoggerFactory.getLogger(WALEntryHandler.class);
-
- private long memTableId = -1;
-
- // cached value, null after this value is flushed to wal successfully
- @SuppressWarnings("squid:S3077")
- private volatile WALEntryValue value;
-
- // wal entry's position in the wal, valid after the value is flushed to wal successfully
- // it's safe to use volatile here to make this reference thread-safe.
- @SuppressWarnings("squid:S3077")
- private final WALEntryPosition walEntryPosition = new WALEntryPosition();
-
- // wal node, null when wal is disabled
- private WALNode walNode = null;
-
- private volatile boolean isHardlink = false;
- private final AtomicReference<File> hardlinkFile = new AtomicReference<>();
-
- public WALEntryHandler(final WALEntryValue value) {
- this.value = value;
- }
-
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable() throws MemTablePinException {
- if (walNode == null || memTableId < 0) {
- throw new MemTablePinException("Fail to pin memTable because of internal error.");
- }
- walNode.pinMemTable(memTableId);
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable() throws MemTablePinException {
- if (walNode == null || memTableId < 0) {
- throw new MemTablePinException("Fail to pin memTable because of internal error.");
- }
- walNode.unpinMemTable(memTableId);
- }
-
- public InsertNode getInsertNodeViaCacheIfPossible() {
- try {
- final WALEntryValue finalValue = value;
- if (finalValue instanceof InsertNode) {
- return (InsertNode) finalValue;
- }
- final Pair<ByteBuffer, InsertNode> byteBufferInsertNodePair =
- walEntryPosition.getByteBufferOrInsertNodeIfPossible();
- return byteBufferInsertNodePair == null ? null : byteBufferInsertNodePair.getRight();
- } catch (final Exception e) {
- logger.warn("Fail to get insert node via cache. {}", this, e);
- throw e;
- }
- }
-
- /**
- * Get this handler's value.
- *
- * @throws WALPipeException when failing to get the value.
- */
- public InsertNode getInsertNode() throws WALPipeException {
- // return local cache
- final WALEntryValue res = value;
- if (res != null) {
- if (res instanceof InsertNode) {
- return (InsertNode) res;
- } else {
- throw new WALPipeException("Fail to get value because the entry type isn't InsertNode.");
- }
- }
-
- // wait until the position is ready
- while (!walEntryPosition.canRead()) {
- try {
- synchronized (this) {
- this.wait();
- }
- } catch (final InterruptedException e) {
- logger.warn("Interrupted when waiting for result.", e);
- Thread.currentThread().interrupt();
- }
- }
-
- final InsertNode node = isHardlink ? readFromHardlinkWALFile() : readFromOriginalWALFile();
- if (node == null) {
- throw new WALPipeException(
- String.format("Fail to get the wal value of the position %s.", walEntryPosition));
- }
- return node;
- }
-
- public ByteBuffer getByteBuffer() throws WALPipeException {
- // wait until the position is ready
- while (!walEntryPosition.canRead()) {
- try {
- synchronized (this) {
- this.wait();
- }
- } catch (InterruptedException e) {
- logger.warn("Interrupted when waiting for result.", e);
- Thread.currentThread().interrupt();
- }
- }
-
- final ByteBuffer buffer = readByteBufferFromWALFile();
- if (buffer == null) {
- throw new WALPipeException(
- String.format("Fail to get the wal value of the position %s.", walEntryPosition));
- }
- return buffer;
- }
-
- private InsertNode readFromOriginalWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readInsertNodeViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- private InsertNode readFromHardlinkWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readInsertNodeViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- private ByteBuffer readByteBufferFromWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readByteBufferViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- public void setWalNode(final WALNode walNode, final long memTableId) {
- this.walNode = walNode;
- this.memTableId = memTableId;
- walEntryPosition.setWalNode(walNode, memTableId);
- }
-
- public long getMemTableId() {
- return memTableId;
- }
-
- public void setEntryPosition(final long walFileVersionId, final long position) {
- this.walEntryPosition.setEntryPosition(walFileVersionId, position, value);
- this.value = null;
- synchronized (this) {
- this.notifyAll();
- }
- }
-
- public WALEntryPosition getWalEntryPosition() {
- return walEntryPosition;
- }
-
- public int getSize() {
- return walEntryPosition.getSize();
- }
-
- public void setSize(final int size) {
- this.walEntryPosition.setSize(size);
- }
-
- public void hardlinkTo(File hardlinkFile) {
- isHardlink = true;
- this.hardlinkFile.set(hardlinkFile);
- }
-
- @Override
- public String toString() {
- return "WALEntryHandler{"
- + "memTableId="
- + memTableId
- + ", value="
- + value
- + ", walEntryPosition="
- + walEntryPosition
- + '}';
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
deleted file mode 100644
index 4d71cb1030c..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALInputStream;
-import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
-
-import org.apache.tsfile.utils.Pair;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.StandardOpenOption;
-import java.util.Objects;
-
-/**
- * This class uses the tuple(identifier, file, position) to denote the position of the wal entry,
- * and give some methods to read the content from the disk.
- */
-public class WALEntryPosition {
- private volatile String identifier = "";
- private volatile long walFileVersionId = -1;
- private volatile long position;
- private volatile int size;
- // wal node, null when wal is disabled
- private WALNode walNode = null;
- // wal file is not null when openReadFileChannel method has been called
- private File walFile = null;
- // cache for wal entry
- private WALInsertNodeCache cache = null;
-
- private static final String ENTRY_NOT_READY_MESSAGE = "This entry isn't ready for read.";
-
- public WALEntryPosition() {}
-
- public WALEntryPosition(String identifier, long walFileVersionId, long position, int size) {
- this.identifier = identifier;
- this.walFileVersionId = walFileVersionId;
- this.position = position;
- this.size = size;
- }
-
- /**
- * Try to read the wal entry directly from the cache. No need to check if the wal entry is ready
- * for read.
- */
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNodeIfPossible() {
- return cache.getByteBufferOrInsertNodeIfPossible(this);
- }
-
- /**
- * Read the wal entry and parse it to the InsertNode. Use LRU cache to accelerate read.
- *
- * @throws IOException failing to read.
- */
- public InsertNode readInsertNodeViaCacheAfterCanRead() throws IOException {
- if (!canRead()) {
- throw new IOException(ENTRY_NOT_READY_MESSAGE);
- }
- return cache.getInsertNode(this);
- }
-
- /**
- * Read the wal entry and get the raw bytebuffer. Use LRU cache to accelerate read.
- *
- * @throws IOException failing to read.
- */
- public ByteBuffer readByteBufferViaCacheAfterCanRead() throws IOException {
- if (!canRead()) {
- throw new IOException(ENTRY_NOT_READY_MESSAGE);
- }
- return cache.getByteBuffer(this);
- }
-
- /**
- * Read the byte buffer directly.
- *
- * @throws IOException failing to read.
- */
- ByteBuffer read() throws IOException {
- if (!canRead()) {
- throw new IOException("Target file hasn't been specified.");
- }
- // TODO: Reuse the file stream
- try (WALInputStream is = openReadFileStream()) {
- is.skipToGivenLogicalPosition(position);
- ByteBuffer buffer = ByteBuffer.allocate(size);
- is.read(buffer);
- return buffer;
- }
- }
-
- /**
- * Open the read file channel for this wal entry, this method will retry automatically when the
- * file is sealed when opening the file channel.
- *
- * @throws IOException failing to open the file channel.
- */
- public FileChannel openReadFileChannel() throws IOException {
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } else {
- try {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } catch (IOException e) {
- // unsealed file may be renamed after sealed, so we should try again
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } else {
- throw e;
- }
- }
- }
- }
-
- public WALInputStream openReadFileStream() throws IOException {
- // TODO: Refactor this part of code
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } else {
- try {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } catch (IOException e) {
- // unsealed file may be renamed after sealed, so we should try again
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } else {
- throw e;
- }
- }
- }
- }
-
- public File getWalFile() {
- return walFile;
- }
-
- /** Return true only when the tuple(file, position, size) is ready. */
- public boolean canRead() {
- return walFileVersionId >= 0;
- }
-
- /** Return true only when this wal file is sealed. */
- public boolean isInSealedFile() {
- if (walNode == null || !canRead()) {
- throw new RuntimeException(ENTRY_NOT_READY_MESSAGE);
- }
- return walFileVersionId < walNode.getCurrentWALFileVersion();
- }
-
- public void setWalNode(WALNode walNode, long memTableId) {
- this.walNode = walNode;
- identifier = walNode.getIdentifier();
- cache = WALInsertNodeCache.getInstance();
- }
-
- public String getIdentifier() {
- return identifier;
- }
-
- public void setEntryPosition(long walFileVersionId, long position, WALEntryValue value) {
- this.position = position;
- this.walFileVersionId = walFileVersionId;
- if (cache != null && value instanceof InsertNode) {
- cache.cacheInsertNodeIfNeeded(this, (InsertNode) value);
- }
- }
-
- public long getPosition() {
- return position;
- }
-
- public long getWalFileVersionId() {
- return walFileVersionId;
- }
-
- public void setSize(int size) {
- this.size = size;
- }
-
- public int getSize() {
- return size;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(identifier, walFileVersionId, position);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- WALEntryPosition that = (WALEntryPosition) o;
- return identifier.equals(that.identifier)
- && walFileVersionId == that.walFileVersionId
- && position == that.position;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
deleted file mode 100644
index f69dc4a0826..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlockType;
-import org.apache.iotdb.db.pipe.resource.memory.PipeModelFixedMemoryBlock;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType;
-import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALByteBufReader;
-
-import com.github.benmanes.caffeine.cache.CacheLoader;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import com.github.benmanes.caffeine.cache.Weigher;
-import org.apache.tsfile.utils.Pair;
-import org.checkerframework.checker.nullness.qual.NonNull;
-import org.checkerframework.checker.nullness.qual.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/** This cache is used by {@link WALEntryPosition}. */
-public class WALInsertNodeCache {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WALInsertNodeCache.class);
- private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
-
- private static PipeModelFixedMemoryBlock walModelFixedMemory = null;
-
- // LRU cache, find Pair<ByteBuffer, InsertNode> by WALEntryPosition
- private final LoadingCache<WALEntryPosition, Pair<ByteBuffer, InsertNode>> lruCache;
-
- // ids of all pinned memTables
- private final Set<Long> memTablesNeedSearch = ConcurrentHashMap.newKeySet();
-
- private volatile boolean hasPipeRunning = false;
-
- private WALInsertNodeCache() {
- if (walModelFixedMemory == null) {
- init();
- }
-
- final long requestedAllocateSize =
- (long)
- (PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- * PIPE_CONFIG.getPipeDataStructureWalMemoryProportion());
-
- lruCache =
- Caffeine.newBuilder()
- .maximumWeight(requestedAllocateSize)
- .weigher(
- (Weigher<WALEntryPosition, Pair<ByteBuffer, InsertNode>>)
- (position, pair) -> {
- long weightInLong = 0L;
- if (pair.right != null) {
- weightInLong = InsertNodeMemoryEstimator.sizeOf(pair.right);
- } else {
- weightInLong = position.getSize();
- }
- if (weightInLong <= 0) {
- return Integer.MAX_VALUE;
- }
- final int weightInInt = (int) weightInLong;
- return weightInInt != weightInLong ? Integer.MAX_VALUE : weightInInt;
- })
- .recordStats()
- .build(new WALInsertNodeCacheLoader());
- }
-
- // please call this method at PipeLauncher
- public static void init() {
- if (walModelFixedMemory != null) {
- return;
- }
- try {
- // Allocate memory for the fixed memory block of WAL
- walModelFixedMemory =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(
- (long)
- (PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- * PIPE_CONFIG.getPipeDataStructureWalMemoryProportion()),
- PipeMemoryBlockType.WAL);
- } catch (Exception e) {
- LOGGER.error("Failed to initialize WAL model fixed memory block", e);
- walModelFixedMemory =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(0, PipeMemoryBlockType.WAL);
- }
- }
-
- /////////////////////////// Getter & Setter ///////////////////////////
-
- public InsertNode getInsertNode(final WALEntryPosition position) {
- final Pair<ByteBuffer, InsertNode> pair = getByteBufferOrInsertNode(position);
-
- if (pair.getRight() != null) {
- return pair.getRight();
- }
-
- if (pair.getLeft() == null) {
- throw new IllegalStateException();
- }
-
- try {
- // multi pipes may share the same wal entry, so we need to wrap the byte[] into
- // different ByteBuffer for each pipe
- final InsertNode insertNode = parse(ByteBuffer.wrap(pair.getLeft().array()));
- pair.setRight(insertNode);
- return insertNode;
- } catch (final Exception e) {
- LOGGER.error(
- "Parsing failed when recovering insertNode from wal, walFile:{}, position:{}, size:{}, exception:",
- position.getWalFile(),
- position.getPosition(),
- position.getSize(),
- e);
- throw e;
- }
- }
-
- private InsertNode parse(final ByteBuffer buffer) {
- final PlanNode node = WALEntry.deserializeForConsensus(buffer);
- if (node instanceof InsertNode) {
- return (InsertNode) node;
- } else {
- return null;
- }
- }
-
- public ByteBuffer getByteBuffer(final WALEntryPosition position) {
- Pair<ByteBuffer, InsertNode> pair = getByteBufferOrInsertNode(position);
-
- if (pair.getLeft() != null) {
- // multi pipes may share the same wal entry, so we need to wrap the byte[] into
- // different ByteBuffer for each pipe
- return ByteBuffer.wrap(pair.getLeft().array());
- }
-
- // forbid multi threads to invalidate and load the same entry
- synchronized (this) {
- lruCache.invalidate(position);
- pair = getByteBufferOrInsertNode(position);
- }
-
- if (pair.getLeft() == null) {
- throw new IllegalStateException();
- }
-
- return ByteBuffer.wrap(pair.getLeft().array());
- }
-
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNode(final WALEntryPosition position) {
- hasPipeRunning = true;
-
- final Pair<ByteBuffer, InsertNode> pair = lruCache.get(position);
-
- if (pair == null) {
- throw new IllegalStateException();
- }
-
- return pair;
- }
-
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNodeIfPossible(
- final WALEntryPosition position) {
- hasPipeRunning = true;
- return lruCache.getIfPresent(position);
- }
-
- public void cacheInsertNodeIfNeeded(
- final WALEntryPosition walEntryPosition, final InsertNode insertNode) {
- // reduce memory usage
- if (hasPipeRunning) {
- lruCache.put(walEntryPosition, new Pair<>(null, insertNode));
- }
- }
-
- //////////////////////////// APIs provided for metric framework ////////////////////////////
-
- public double getCacheHitRate() {
- return Objects.nonNull(lruCache) ? lruCache.stats().hitRate() : 0;
- }
-
- public double getCacheHitCount() {
- return Objects.nonNull(lruCache) ? lruCache.stats().hitCount() : 0;
- }
-
- public double getCacheRequestCount() {
- return Objects.nonNull(lruCache) ? lruCache.stats().requestCount() : 0;
- }
-
- /////////////////////////// MemTable ///////////////////////////
-
- public void addMemTable(final long memTableId) {
- memTablesNeedSearch.add(memTableId);
- }
-
- public void removeMemTable(final long memTableId) {
- memTablesNeedSearch.remove(memTableId);
- }
-
- /////////////////////////// Cache Loader ///////////////////////////
-
- class WALInsertNodeCacheLoader
- implements CacheLoader<WALEntryPosition, Pair<ByteBuffer, InsertNode>> {
-
- @Override
- public @Nullable Pair<ByteBuffer, InsertNode> load(@NonNull final WALEntryPosition key)
- throws Exception {
- return new Pair<>(key.read(), null);
- }
-
- /** Batch load all wal entries in the file when any one key is absent. */
- @Override
- public @NonNull Map<@NonNull WALEntryPosition, @NonNull Pair<ByteBuffer, InsertNode>> loadAll(
- @NonNull final Iterable<? extends @NonNull WALEntryPosition> walEntryPositions) {
- final Map<WALEntryPosition, Pair<ByteBuffer, InsertNode>> loadedEntries = new HashMap<>();
-
- for (final WALEntryPosition walEntryPosition : walEntryPositions) {
- if (loadedEntries.containsKey(walEntryPosition) || !walEntryPosition.canRead()) {
- continue;
- }
-
- final long walFileVersionId = walEntryPosition.getWalFileVersionId();
-
- // load one when wal file is not sealed
- if (!walEntryPosition.isInSealedFile()) {
- try {
- loadedEntries.put(walEntryPosition, load(walEntryPosition));
- } catch (final Exception e) {
- LOGGER.info(
- "Fail to cache wal entries from the wal file with version id {}",
- walFileVersionId,
- e);
- }
- continue;
- }
-
- // batch load when wal file is sealed
- long position = 0;
- try (final WALByteBufReader walByteBufReader = new WALByteBufReader(walEntryPosition)) {
- while (walByteBufReader.hasNext()) {
- // see WALInfoEntry#serialize, entry type + memtable id + plan node type
- final ByteBuffer buffer = walByteBufReader.next();
-
- final int size = buffer.capacity();
- final WALEntryType type = WALEntryType.valueOf(buffer.get());
- final long memTableId = buffer.getLong();
-
- if ((memTablesNeedSearch.contains(memTableId)
- || walEntryPosition.getPosition() == position)
- && type.needSearch()) {
- buffer.clear();
- loadedEntries.put(
- new WALEntryPosition(
- walEntryPosition.getIdentifier(), walFileVersionId, position, size),
- new Pair<>(buffer, null));
- }
-
- position += size;
- }
- } catch (final IOException e) {
- LOGGER.info(
- "Fail to cache wal entries from the wal file with version id {}",
- walFileVersionId,
- e);
- }
- }
-
- return loadedEntries;
- }
- }
-
- /////////////////////////// Singleton ///////////////////////////
-
- public static WALInsertNodeCache getInstance() {
- return InstanceHolder.INSTANCE;
- }
-
- private static class InstanceHolder {
-
- public static final WALInsertNodeCache INSTANCE = new WALInsertNodeCache();
-
- private InstanceHolder() {
- // forbidding instantiation
- }
- }
-
- /////////////////////////// Test Only ///////////////////////////
-
- @TestOnly
- boolean contains(WALEntryPosition position) {
- return lruCache.getIfPresent(position) != null;
- }
-
- @TestOnly
- public void clear() {
- lruCache.invalidateAll();
- memTablesNeedSearch.clear();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
index 8c84a0cb0fe..7896ea01dcf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
@@ -19,20 +19,10 @@
package org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
/** This class helps judge whether wal is flushed to the storage device. */
public class WALFlushListener extends AbstractResultListener {
- // handler for pipeline, only exists when value is InsertNode
- private final WALEntryHandler walEntryHandler;
- public WALFlushListener(boolean wait, WALEntryValue value) {
+ public WALFlushListener(boolean wait) {
super(wait);
- walEntryHandler = new WALEntryHandler(value);
- }
-
- public WALEntryHandler getWalEntryHandler() {
- return walEntryHandler;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 25b594631aa..28c952b3d72 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -505,6 +505,7 @@ public class LoadTsFileManager {
final DataRegion dataRegion = entry.getKey().getDataRegion();
final TsFileResource tsFileResource = dataPartition2Resource.get(entry.getKey());
+ tsFileResource.setGeneratedByPipe(isGeneratedByPipe);
endTsFileResource(
writer,
tsFileResource,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
index 7dfe91ad62e..cb147f9c286 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
@@ -20,9 +20,6 @@
package org.apache.iotdb.db.storageengine.load.converter;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
-import org.apache.iotdb.commons.concurrent.ThreadName;
-import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBPipePattern;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReq;
@@ -48,14 +45,6 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil.calculateTabletSizeInBytes;
@@ -70,9 +59,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
.getConfig()
.getLoadTsFileTabletConversionBatchMemorySizeInBytes();
- private static final AtomicReference<WrappedThreadPoolExecutor> executorPool =
- new AtomicReference<>();
-
private final StatementExecutor statementExecutor;
@FunctionalInterface
@@ -80,21 +66,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
TSStatus execute(final Statement statement);
}
- public static class CallerBlocksPolicy implements RejectedExecutionHandler {
- public CallerBlocksPolicy() {}
-
- public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
- if (!e.isShutdown()) {
- try {
- e.getQueue().put(r);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new RejectedExecutionException("task " + r + " rejected from " + e, ie);
- }
- }
- }
- }
-
public LoadTreeStatementDataTypeConvertExecutionVisitor(
final StatementExecutor statementExecutor) {
this.statementExecutor = statementExecutor;
@@ -118,7 +89,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
final List<Long> tabletRawReqSizes = new ArrayList<>();
try {
- final List<Future<TSStatus>> executionFutures = new ArrayList<>();
for (final File file : loadTsFileStatement.getTsFiles()) {
try (final TsFileInsertionScanDataContainer container =
new TsFileInsertionScanDataContainer(
@@ -136,16 +106,9 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
continue;
}
- final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
- batchStatement.setInsertTabletStatementList(
- tabletRawReqs.stream()
- .map(
- req ->
- new LoadConvertedInsertTabletStatement(
- req.constructStatement(),
- loadTsFileStatement.isConvertOnTypeMismatch()))
- .collect(Collectors.toList()));
- executionFutures.add(executeInsertMultiTabletsWithRetry(batchStatement));
+ final TSStatus result =
+ executeInsertMultiTabletsWithRetry(
+ tabletRawReqs, loadTsFileStatement.isConvertOnTypeMismatch());
for (final long memoryCost : tabletRawReqSizes) {
block.reduceMemoryUsage(memoryCost);
@@ -153,6 +116,10 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
tabletRawReqs.clear();
tabletRawReqSizes.clear();
+ if (!handleTSStatus(result, loadTsFileStatement)) {
+ return Optional.empty();
+ }
+
tabletRawReqs.add(tabletRawReq);
tabletRawReqSizes.add(curMemory);
block.addMemoryUsage(curMemory);
@@ -166,36 +133,22 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
if (!tabletRawReqs.isEmpty()) {
try {
- final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
- batchStatement.setInsertTabletStatementList(
- tabletRawReqs.stream()
- .map(
- req ->
- new LoadConvertedInsertTabletStatement(
- req.constructStatement(),
- loadTsFileStatement.isConvertOnTypeMismatch()))
- .collect(Collectors.toList()));
- executionFutures.add(executeInsertMultiTabletsWithRetry(batchStatement));
+ final TSStatus result =
+ executeInsertMultiTabletsWithRetry(
+ tabletRawReqs, loadTsFileStatement.isConvertOnTypeMismatch());
for (final long memoryCost : tabletRawReqSizes) {
block.reduceMemoryUsage(memoryCost);
}
tabletRawReqs.clear();
tabletRawReqSizes.clear();
- } catch (final Exception e) {
- LOGGER.warn(
- "Failed to convert data type for LoadTsFileStatement: {}.", loadTsFileStatement, e);
- return Optional.empty();
- }
- }
- for (final Future<TSStatus> future : executionFutures) {
- try {
- if (!handleTSStatus(future.get(), loadTsFileStatement)) {
+ if (!handleTSStatus(result, loadTsFileStatement)) {
return Optional.empty();
}
- } catch (ExecutionException | InterruptedException e) {
- LOGGER.warn("Exception occurs when executing insertion during tablet conversion: ", e);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to convert data type for LoadTsFileStatement: {}.", loadTsFileStatement, e);
return Optional.empty();
}
}
@@ -226,67 +179,43 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
return Optional.of(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
- private Future<TSStatus> executeInsertMultiTabletsWithRetry(
- final InsertMultiTabletsStatement batchStatement) {
- return getExecutorPool()
- .submit(
- () -> {
- TSStatus result;
- try {
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(batchStatement));
-
- // Retry max 5 times if the write process is rejected
- for (int i = 0;
- i < 5
- && result.getCode()
- == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
- i++) {
- Thread.sleep(100L * (i + 1));
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(batchStatement));
- }
- } catch (final Exception e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
- }
- return result;
- });
- }
-
- public static WrappedThreadPoolExecutor getExecutorPool() {
- if (executorPool.get() == null) {
- synchronized (executorPool) {
- if (executorPool.get() == null) {
- executorPool.set(
- new WrappedThreadPoolExecutor(
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount(),
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount(),
- 0L,
- TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount()),
- new IoTThreadFactory(ThreadName.LOAD_DATATYPE_CONVERT_POOL.getName()),
- ThreadName.LOAD_DATATYPE_CONVERT_POOL.getName(),
- new CallerBlocksPolicy()));
- }
+ private TSStatus executeInsertMultiTabletsWithRetry(
+ final List<PipeTransferTabletRawReq> tabletRawReqs, boolean isConvertOnTypeMismatch) {
+ final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
+ batchStatement.setInsertTabletStatementList(
+ tabletRawReqs.stream()
+ .map(
+ req ->
+ new LoadConvertedInsertTabletStatement(
+ req.constructStatement(), isConvertOnTypeMismatch))
+ .collect(Collectors.toList()));
+
+ TSStatus result;
+ try {
+ result =
+ batchStatement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(batchStatement));
+
+ // Retry max 5 times if the write process is rejected
+ for (int i = 0;
+ i < 5
+ && result.getCode()
+ == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
+ i++) {
+ Thread.sleep(100L * (i + 1));
+ result =
+ batchStatement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(batchStatement));
+ }
+ } catch (final Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
}
+ result = batchStatement.accept(LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
}
- return executorPool.get();
+ return result;
}
private static boolean handleTSStatus(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
index a46546e3e36..81a1f0639be 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
@@ -20,14 +20,13 @@
package org.apache.iotdb.db.storageengine.load.converter;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.protocol.session.IClientSession;
-import org.apache.iotdb.db.protocol.session.InternalClientSession;
import org.apache.iotdb.db.protocol.session.SessionManager;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
+import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager;
+import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
@@ -37,7 +36,6 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.ZoneId;
import java.util.Optional;
public class LoadTsFileDataTypeConverter {
@@ -52,16 +50,20 @@ public class LoadTsFileDataTypeConverter {
STATEMENT_EXCEPTION_VISITOR = new LoadConvertedInsertTabletStatementExceptionVisitor();
private final boolean isGeneratedByPipe;
+ private final MPPQueryContext context;
private final LoadTreeStatementDataTypeConvertExecutionVisitor
treeStatementDataTypeConvertExecutionVisitor =
new LoadTreeStatementDataTypeConvertExecutionVisitor(this::executeForTreeModel);
- public LoadTsFileDataTypeConverter(final boolean isGeneratedByPipe) {
+ public LoadTsFileDataTypeConverter(
+ final MPPQueryContext context, final boolean isGeneratedByPipe) {
+ this.context = context;
this.isGeneratedByPipe = isGeneratedByPipe;
}
public Optional<TSStatus> convertForTreeModel(final LoadTsFileStatement loadTsFileTreeStatement) {
+ DataNodeSchemaLockManager.getInstance().releaseReadLock(context);
try {
return loadTsFileTreeStatement.accept(treeStatementDataTypeConvertExecutionVisitor, null);
} catch (Exception e) {
@@ -69,36 +71,24 @@ public class LoadTsFileDataTypeConverter {
"Failed to convert data types for tree model statement {}.", loadTsFileTreeStatement, e);
return Optional.of(
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
+ } finally {
+ DataNodeSchemaLockManager.getInstance()
+ .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION);
}
}
private TSStatus executeForTreeModel(final Statement statement) {
- final IClientSession session =
- new InternalClientSession(
- String.format(
- "%s_%s",
- LoadTsFileDataTypeConverter.class.getSimpleName(),
- Thread.currentThread().getName()));
- session.setUsername(AuthorityChecker.SUPER_USER);
- session.setClientVersion(IoTDBConstant.ClientVersion.V_1_0);
- session.setZoneId(ZoneId.systemDefault());
-
- SESSION_MANAGER.registerSession(session);
- try {
- return Coordinator.getInstance()
- .executeForTreeModel(
- isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement,
- SESSION_MANAGER.requestQueryId(),
- SESSION_MANAGER.getSessionInfo(session),
- "",
- ClusterPartitionFetcher.getInstance(),
- ClusterSchemaFetcher.getInstance(),
- IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(),
- false)
- .status;
- } finally {
- SESSION_MANAGER.removeCurrSession();
- }
+ return Coordinator.getInstance()
+ .executeForTreeModel(
+ isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement,
+ SESSION_MANAGER.requestQueryId(),
+ SESSION_MANAGER.getSessionInfo(SESSION_MANAGER.getCurrSession()),
+ "",
+ ClusterPartitionFetcher.getInstance(),
+ ClusterSchemaFetcher.getInstance(),
+ IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(),
+ false)
+ .status;
}
public boolean isSuccessful(final TSStatus status) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java
index f473a56a447..5dcc5d8943c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java
@@ -26,25 +26,22 @@ import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
import org.apache.iotdb.db.subscription.task.subtask.SubscriptionReceiverSubtask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
public class SubscriptionSubtaskExecutor extends PipeConnectorSubtaskExecutor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionSubtaskExecutor.class);
+ private static final AtomicInteger id = new AtomicInteger(0);
private final AtomicLong submittedReceiverSubtasks = new AtomicLong(0);
public SubscriptionSubtaskExecutor() {
super(
SubscriptionConfig.getInstance().getSubscriptionSubtaskExecutorMaxThreadNum(),
- ThreadName.SUBSCRIPTION_EXECUTOR_POOL);
+ ThreadName.SUBSCRIPTION_EXECUTOR_POOL.getName() + "-" + id.getAndIncrement());
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java
index 003d712cc5b..7d26701004a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java
@@ -36,7 +36,7 @@ public class SubscriptionTaskConnectorStage extends PipeTaskConnectorStage {
PipeParameters pipeConnectorParameters,
int regionId,
PipeConnectorSubtaskExecutor executor) {
- super(pipeName, creationTime, pipeConnectorParameters, regionId, executor);
+ super(pipeName, creationTime, pipeConnectorParameters, regionId, () -> executor);
}
@Override
@@ -44,7 +44,7 @@ public class SubscriptionTaskConnectorStage extends PipeTaskConnectorStage {
this.connectorSubtaskId =
SubscriptionConnectorSubtaskManager.instance()
.register(
- executor,
+ executor.get(),
pipeConnectorParameters,
new PipeTaskConnectorRuntimeEnvironment(pipeName, creationTime, regionId));
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
index 8e69473b95c..a581d111dc6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
@@ -71,7 +71,6 @@ public class PipeDataNodeThriftRequestTest {
... 3520 lines suppressed ...
(iotdb) 11/11: [To dev/1.3] Pipe: Reduced the progress index report interval & Added some logs (#15905) (#15908)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fedfa10fc8d1ca922873c9f3319a6987afc7f72a
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Thu Jul 10 19:24:46 2025 +0800
[To dev/1.3] Pipe: Reduced the progress index report interval & Added some logs (#15905) (#15908)
* partial
* Changed default
* Update PipeDataNodeTaskAgent.java
* Next
---
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 20 ++++++------
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 37 ++++++++++++----------
.../apache/iotdb/commons/conf/CommonConfig.java | 8 ++---
.../iotdb/commons/pipe/config/PipeConfig.java | 2 +-
.../iotdb/commons/pipe/config/PipeDescriptor.java | 2 +-
.../commons/pipe/resource/log/PipeLogManager.java | 2 +-
.../commons/pipe/resource/log/PipeLogStatus.java | 4 +--
7 files changed, 38 insertions(+), 37 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
index 911909d104c..b79fd8da94f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
@@ -207,21 +207,19 @@ public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
if (isShutdown() || !PipeConfigNodeAgent.runtime().isLeaderReady()) {
return;
}
-
- LOGGER.info("Received pipe heartbeat request {} from config coordinator.", req.heartbeatId);
+ final Optional<Logger> logger =
+ PipeConfigNodeResourceManager.log()
+ .schedule(
+ PipeConfigNodeTaskAgent.class,
+ PipeConfig.getInstance().getPipeMetaReportMaxLogNumPerRound(),
+ PipeConfig.getInstance().getPipeMetaReportMaxLogIntervalRounds(),
+ pipeMetaKeeper.getPipeMetaCount());
+ LOGGER.debug("Received pipe heartbeat request {} from config coordinator.", req.heartbeatId);
final List<ByteBuffer> pipeMetaBinaryList = new ArrayList<>();
final List<Long> pipeRemainingEventCountList = new ArrayList<>();
final List<Double> pipeRemainingTimeList = new ArrayList<>();
try {
- final Optional<Logger> logger =
- PipeConfigNodeResourceManager.log()
- .schedule(
- PipeConfigNodeTaskAgent.class,
- PipeConfig.getInstance().getPipeMetaReportMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeMetaReportMaxLogIntervalRounds(),
- pipeMetaKeeper.getPipeMetaCount());
-
for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
pipeMetaBinaryList.add(pipeMeta.serialize());
@@ -244,7 +242,7 @@ public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
remainingEventCount,
estimatedRemainingTime));
}
- LOGGER.info("Reported {} pipe metas.", pipeMetaBinaryList.size());
+ logger.ifPresent(l -> l.info("Reported {} pipe metas.", pipeMetaBinaryList.size()));
} catch (final IOException e) {
throw new TException(e);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
index 15e20347393..f800c15c3f8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
@@ -381,6 +381,13 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
if (PipeDataNodeAgent.runtime().isShutdown()) {
return;
}
+ final Optional<Logger> logger =
+ PipeDataNodeResourceManager.log()
+ .schedule(
+ PipeDataNodeTaskAgent.class,
+ PipeConfig.getInstance().getPipeMetaReportMaxLogNumPerRound(),
+ PipeConfig.getInstance().getPipeMetaReportMaxLogIntervalRounds(),
+ pipeMetaKeeper.getPipeMetaCount());
final Set<Integer> dataRegionIds =
StorageEngine.getInstance().getAllDataRegionIds().stream()
@@ -392,13 +399,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
final List<Long> pipeRemainingEventCountList = new ArrayList<>();
final List<Double> pipeRemainingTimeList = new ArrayList<>();
try {
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeDataNodeTaskAgent.class,
- PipeConfig.getInstance().getPipeMetaReportMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeMetaReportMaxLogIntervalRounds(),
- pipeMetaKeeper.getPipeMetaCount());
for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
pipeMetaBinaryList.add(pipeMeta.serialize());
@@ -445,7 +445,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
remainingEventAndTime.getLeft(),
remainingEventAndTime.getRight()));
}
- LOGGER.info("Reported {} pipe metas.", pipeMetaBinaryList.size());
+ logger.ifPresent(l -> l.info("Reported {} pipe metas.", pipeMetaBinaryList.size()));
} catch (final IOException | IllegalPathException e) {
throw new TException(e);
}
@@ -460,10 +460,18 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
protected void collectPipeMetaListInternal(
final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws TException {
// Do nothing if data node is removing or removed, or request does not need pipe meta list
+ // If the heartbeatId == Long.MIN_VALUE then it's shutdown report and shall not be skipped
if (PipeDataNodeAgent.runtime().isShutdown() && req.heartbeatId != Long.MIN_VALUE) {
return;
}
- LOGGER.info("Received pipe heartbeat request {} from config node.", req.heartbeatId);
+ final Optional<Logger> logger =
+ PipeDataNodeResourceManager.log()
+ .schedule(
+ PipeDataNodeTaskAgent.class,
+ PipeConfig.getInstance().getPipeMetaReportMaxLogNumPerRound(),
+ PipeConfig.getInstance().getPipeMetaReportMaxLogIntervalRounds(),
+ pipeMetaKeeper.getPipeMetaCount());
+ LOGGER.debug("Received pipe heartbeat request {} from config node.", req.heartbeatId);
final Set<Integer> dataRegionIds =
StorageEngine.getInstance().getAllDataRegionIds().stream()
@@ -475,13 +483,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
final List<Long> pipeRemainingEventCountList = new ArrayList<>();
final List<Double> pipeRemainingTimeList = new ArrayList<>();
try {
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeDataNodeTaskAgent.class,
- PipeConfig.getInstance().getPipeMetaReportMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeMetaReportMaxLogIntervalRounds(),
- pipeMetaKeeper.getPipeMetaCount());
for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
pipeMetaBinaryList.add(pipeMeta.serialize());
@@ -528,7 +529,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
remainingEventAndTime.getLeft(),
remainingEventAndTime.getRight()));
}
- LOGGER.info("Reported {} pipe metas.", pipeMetaBinaryList.size());
+ logger.ifPresent(l -> l.info("Reported {} pipe metas.", pipeMetaBinaryList.size()));
} catch (final IOException | IllegalPathException e) {
throw new TException(e);
}
@@ -613,6 +614,8 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
IoTDBDescriptor.getInstance().getConfig().getDataNodeId(), resp);
if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != result.getCode()) {
LOGGER.warn("Failed to persist progress index to configNode, status: {}", result);
+ } else {
+ LOGGER.info("Successfully persisted all pipe's info to configNode.");
}
} catch (final Exception e) {
LOGGER.warn(e.getMessage());
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index dfcbba78cff..3ac297e1c69 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -284,7 +284,7 @@ public class CommonConfig {
(int) (RpcUtils.THRIFT_FRAME_MAX_SIZE * 0.8);
private boolean isSeperatedPipeHeartbeatEnabled = true;
- private int pipeHeartbeatIntervalSecondsForCollectingPipeMeta = 30;
+ private int pipeHeartbeatIntervalSecondsForCollectingPipeMeta = 3;
private long pipeMetaSyncerInitialSyncDelayMinutes = 3;
private long pipeMetaSyncerSyncIntervalMinutes = 3;
private long pipeMetaSyncerAutoRestartPipeCheckIntervalRound = 1;
@@ -303,8 +303,8 @@ public class CommonConfig {
private int pipeMaxAllowedPendingTsFileEpochPerDataRegion = Integer.MAX_VALUE; // Deprecated
private long pipeMaxAllowedLinkedTsFileCount = Long.MAX_VALUE; // Deprecated
- private int pipeMetaReportMaxLogNumPerRound = 10;
- private int pipeMetaReportMaxLogIntervalRounds = 36;
+ private double pipeMetaReportMaxLogNumPerRound = 0.1;
+ private int pipeMetaReportMaxLogIntervalRounds = 360;
private int pipeTsFilePinMaxLogNumPerRound = 10;
private int pipeTsFilePinMaxLogIntervalRounds = 90;
@@ -1581,7 +1581,7 @@ public class CommonConfig {
return pipeMetaReportMaxLogNumPerRound;
}
- public void setPipeMetaReportMaxLogNumPerRound(int pipeMetaReportMaxLogNumPerRound) {
+ public void setPipeMetaReportMaxLogNumPerRound(double pipeMetaReportMaxLogNumPerRound) {
if (this.pipeMetaReportMaxLogNumPerRound == pipeMetaReportMaxLogNumPerRound) {
return;
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
index 4ce2fd19093..25fe88d72c9 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
@@ -347,7 +347,7 @@ public class PipeConfig {
/////////////////////////////// Logger ///////////////////////////////
- public int getPipeMetaReportMaxLogNumPerRound() {
+ public double getPipeMetaReportMaxLogNumPerRound() {
return COMMON_CONFIG.getPipeMetaReportMaxLogNumPerRound();
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
index d6bad9d18f6..9e13bb04ade 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
@@ -118,7 +118,7 @@ public class PipeDescriptor {
Integer.toString(config.getPipeAirGapReceiverPort()))));
config.setPipeMetaReportMaxLogNumPerRound(
- Integer.parseInt(
+ Double.parseDouble(
properties.getProperty(
"pipe_meta_report_max_log_num_per_round",
String.valueOf(config.getPipeMetaReportMaxLogNumPerRound()))));
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogManager.java
index 49699fdf878..69d8b5294db 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogManager.java
@@ -34,7 +34,7 @@ public class PipeLogManager {
public Optional<Logger> schedule(
final Class<?> logClass,
- final int maxAverageScale,
+ final double maxAverageScale,
final int maxLogInterval,
final int scale) {
return logClass2LogStatusMap
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogStatus.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogStatus.java
index 3b4db49d065..a30b12ab5eb 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogStatus.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/log/PipeLogStatus.java
@@ -29,11 +29,11 @@ class PipeLogStatus {
private final Logger logger;
- private final int maxAverageScale;
+ private final double maxAverageScale;
private final int maxLogInterval;
private final AtomicLong currentRounds = new AtomicLong(0);
- PipeLogStatus(final Class<?> logClass, final int maxAverageScale, final int maxLogInterval) {
+ PipeLogStatus(final Class<?> logClass, final double maxAverageScale, final int maxLogInterval) {
logger = LoggerFactory.getLogger(logClass);
this.maxAverageScale = maxAverageScale;
(iotdb) 03/11: Pipe: Added rate limiter for tsFile sending (#15765) (#15947)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2e339af15f6f5fd132f24bf9c3aec39197ad266c
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Tue Jul 15 23:50:22 2025 +0800
Pipe: Added rate limiter for tsFile sending (#15765) (#15947)
As the title said.
---
.../protocol/IoTDBConfigRegionAirGapConnector.java | 5 ++
.../protocol/IoTDBConfigRegionConnector.java | 5 ++
.../task/builder/PipeDataNodeTaskBuilder.java | 68 ++++++++++++++--------
.../airgap/IoTDBDataRegionAirGapConnector.java | 31 ++++++++++
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 5 ++
.../async/IoTDBDataRegionAsyncConnector.java | 14 +++++
.../async/handler/PipeTransferTsFileHandler.java | 6 ++
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 21 +++++++
.../thrift/sync/IoTDBSchemaRegionConnector.java | 5 ++
.../dataregion/IoTDBDataRegionExtractor.java | 4 +-
.../pipe/metric/overview/PipeResourceMetrics.java | 14 ++++-
.../load/limiter/LoadTsFileRateLimiter.java | 64 +++-----------------
.../apache/iotdb/commons/conf/CommonConfig.java | 16 +++++
.../iotdb/commons/pipe/config/PipeConfig.java | 6 ++
.../iotdb/commons/pipe/config/PipeDescriptor.java | 6 ++
.../config/constant/PipeConnectorConstant.java | 5 ++
.../connector/limiter/GlobalRPCRateLimiter.java | 33 +++++++++++
.../pipe/connector/limiter/GlobalRateLimiter.java | 14 ++---
.../connector/limiter/TsFileSendRateLimiter.java | 47 +++++++++++++++
.../connector/protocol/IoTDBAirGapConnector.java | 3 +
.../pipe/connector/protocol/IoTDBConnector.java | 4 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 3 +
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
23 files changed, 289 insertions(+), 91 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java
index c5fdc4765f9..595182d47d5 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java
@@ -85,6 +85,11 @@ public class IoTDBConfigRegionAirGapConnector extends IoTDBAirGapConnector {
return PipeTransferConfigNodeHandshakeV2Req.toTPipeTransferBytes(params);
}
+ @Override
+ protected void mayLimitRateAndRecordIO(final long requiredBytes) {
+ // Do nothing
+ }
+
@Override
protected boolean mayNeedHandshakeWhenFail() {
return true;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java
index 3761f696875..f04e18027b0 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java
@@ -97,6 +97,11 @@ public class IoTDBConfigRegionConnector extends IoTDBSslSyncConnector {
return PipeTransferConfigSnapshotPieceReq.toTPipeTransferReq(fileName, position, payLoad);
}
+ @Override
+ protected void mayLimitRateAndRecordIO(final long requiredBytes) {
+ // Do nothing
+ }
+
@Override
public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception {
throw new UnsupportedOperationException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
index f84cb73fd69..6ba1acae9bb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
@@ -52,7 +52,10 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstan
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY;
public class PipeDataNodeTaskBuilder {
@@ -170,10 +173,6 @@ public class PipeDataNodeTaskBuilder {
extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_SNAPSHOT_VALUE)
|| extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_QUERY_VALUE);
- if (!insertionDeletionListeningOptionPair.right
- && !shouldTerminatePipeOnAllHistoricalEventsConsumed) {
- return;
- }
} catch (final IllegalPathException e) {
LOGGER.warn(
"PipeDataNodeTaskBuilder failed to parse 'inclusion' and 'exclusion' parameters: {}",
@@ -182,29 +181,52 @@ public class PipeDataNodeTaskBuilder {
return;
}
- final Boolean isRealtime =
- connectorParameters.getBooleanByKeys(
- PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY,
- PipeConnectorConstant.SINK_REALTIME_FIRST_KEY);
- if (isRealtime == null) {
- connectorParameters.addAttribute(PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY, "false");
- if (insertionDeletionListeningOptionPair.right) {
- LOGGER.info(
- "PipeDataNodeTaskBuilder: When 'inclusion' contains 'data.delete', 'realtime-first' is defaulted to 'false' to prevent sync issues after deletion.");
- } else {
- LOGGER.info(
- "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' is defaulted to 'false' to prevent premature halt before transfer completion.");
+ if (insertionDeletionListeningOptionPair.right
+ || shouldTerminatePipeOnAllHistoricalEventsConsumed) {
+ final Boolean isRealtime =
+ connectorParameters.getBooleanByKeys(
+ PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY,
+ PipeConnectorConstant.SINK_REALTIME_FIRST_KEY);
+ if (isRealtime == null) {
+ connectorParameters.addAttribute(
+ PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY, "false");
+ if (insertionDeletionListeningOptionPair.right) {
+ LOGGER.info(
+ "PipeDataNodeTaskBuilder: When 'inclusion' contains 'data.delete', 'realtime-first' is defaulted to 'false' to prevent sync issues after deletion.");
+ } else {
+ LOGGER.info(
+ "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' is defaulted to 'false' to prevent premature halt before transfer completion.");
+ }
+ } else if (isRealtime) {
+ if (insertionDeletionListeningOptionPair.right) {
+ LOGGER.warn(
+ "PipeDataNodeTaskBuilder: When 'inclusion' includes 'data.delete', 'realtime-first' set to 'true' may result in data synchronization issues after deletion.");
+ } else {
+ LOGGER.warn(
+ "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' set to 'true' may cause prevent premature halt before transfer completion.");
+ }
}
- return;
}
- if (isRealtime) {
- if (insertionDeletionListeningOptionPair.right) {
- LOGGER.warn(
- "PipeDataNodeTaskBuilder: When 'inclusion' includes 'data.delete', 'realtime-first' set to 'true' may result in data synchronization issues after deletion.");
- } else {
+ final boolean isRealtimeEnabled =
+ extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
+ EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE);
+
+ if (isRealtimeEnabled && !shouldTerminatePipeOnAllHistoricalEventsConsumed) {
+ final Boolean enableSendTsFileLimit =
+ connectorParameters.getBooleanByKeys(
+ PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT,
+ PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT);
+
+ if (enableSendTsFileLimit == null) {
+ connectorParameters.addAttribute(
+ PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT, "true");
+ LOGGER.info(
+ "PipeDataNodeTaskBuilder: When the realtime sync is enabled, we enable rate limiter in sending tsfile by default to reserve disk and network IO for realtime sending.");
+ } else if (!enableSendTsFileLimit) {
LOGGER.warn(
- "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' set to 'true' may cause prevent premature halt before transfer completion.");
+ "PipeDataNodeTaskBuilder: When the realtime sync is enabled, not enabling the rate limiter in sending tsfile may introduce delay for realtime sending.");
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
index bc4805e8c8e..57b92262abb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.pipe.connector.protocol.airgap;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.pipe.connector.limiter.TsFileSendRateLimiter;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletBinaryReq;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletInsertNodeReq;
@@ -34,9 +35,12 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertio
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
+import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
+import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
@@ -49,13 +53,32 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Objects;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT;
+
public class IoTDBDataRegionAirGapConnector extends IoTDBDataNodeAirGapConnector {
private static final Logger LOGGER =
LoggerFactory.getLogger(IoTDBDataRegionAirGapConnector.class);
+ private boolean enableSendTsFileLimit;
+
+ @Override
+ public void customize(
+ final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration)
+ throws Exception {
+ super.customize(parameters, configuration);
+
+ enableSendTsFileLimit =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(SINK_ENABLE_SEND_TSFILE_LIMIT, CONNECTOR_ENABLE_SEND_TSFILE_LIMIT),
+ CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE);
+ }
+
@Override
public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception {
// PipeProcessor can change the type of TabletInsertionEvent
@@ -293,6 +316,14 @@ public class IoTDBDataRegionAirGapConnector extends IoTDBDataNodeAirGapConnector
}
}
+ @Override
+ protected void mayLimitRateAndRecordIO(final long requiredBytes) {
+ PipeResourceMetrics.getInstance().recordDiskIO(requiredBytes);
+ if (enableSendTsFileLimit) {
+ TsFileSendRateLimiter.getInstance().acquire(requiredBytes);
+ }
+ }
+
@Override
protected byte[] getTransferSingleFilePieceBytes(
final String fileName, final long position, final byte[] payLoad) throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java
index 342807a5dfb..84cc085509c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java
@@ -143,6 +143,11 @@ public class IoTDBSchemaRegionAirGapConnector extends IoTDBDataNodeAirGapConnect
}
}
+ @Override
+ protected void mayLimitRateAndRecordIO(final long requiredBytes) {
+ // Do nothing
+ }
+
@Override
protected byte[] getTransferSingleFilePieceBytes(
final String fileName, final long position, final byte[] payLoad) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
index 0dacf0cb912..f78a52a8281 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
@@ -78,8 +78,11 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_SSL_ENABLE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY;
@@ -115,6 +118,8 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
private final Map<PipeTransferTrackableHandler, PipeTransferTrackableHandler> pendingHandlers =
new ConcurrentHashMap<>();
+ private boolean enableSendTsFileLimit;
+
@Override
public void validate(final PipeParameterValidator validator) throws Exception {
super.validate(validator);
@@ -170,6 +175,11 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
if (isTabletBatchModeEnabled) {
tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters);
}
+
+ enableSendTsFileLimit =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(SINK_ENABLE_SEND_TSFILE_LIMIT, CONNECTOR_ENABLE_SEND_TSFILE_LIMIT),
+ CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE);
}
@Override
@@ -682,6 +692,10 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
events.forEach(this::addFailureEventToRetryQueue);
}
+ public boolean isEnableSendTsFileLimit() {
+ return enableSendTsFileLimit;
+ }
+
//////////////////////////// Operations for close ////////////////////////////
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
index 7353ea91e91..dbbf489b5a5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.pipe.connector.protocol.thrift.async.handler;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import org.apache.iotdb.commons.pipe.connector.limiter.TsFileSendRateLimiter;
import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.utils.RetryUtils;
@@ -32,6 +33,7 @@ import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransfer
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFileSealWithModReq;
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeTsFileMemoryBlock;
@@ -164,6 +166,10 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
client.setShouldReturnSelf(false);
client.setTimeoutDynamically(clientManager.getConnectionTimeout());
+ PipeResourceMetrics.getInstance().recordDiskIO(readFileBufferSize);
+ if (connector.isEnableSendTsFileLimit()) {
+ TsFileSendRateLimiter.getInstance().acquire(readFileBufferSize);
+ }
final int readLength = reader.read(readBuffer);
if (readLength == -1) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
index d996cac2820..e0fd0b11827 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.pipe.connector.protocol.thrift.sync;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.pipe.connector.client.IoTDBSyncClient;
+import org.apache.iotdb.commons.pipe.connector.limiter.TsFileSendRateLimiter;
import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferFilePieceReq;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.utils.RetryUtils;
@@ -43,6 +44,7 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertio
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
@@ -65,16 +67,22 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT;
+
public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionSyncConnector.class);
private PipeTransferBatchReqBuilder tabletBatchBuilder;
+ private boolean enableSendTsFileLimit;
@Override
public void customize(
@@ -86,6 +94,11 @@ public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
if (isTabletBatchModeEnabled) {
tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters);
}
+
+ enableSendTsFileLimit =
+ parameters.getBooleanOrDefault(
+ Arrays.asList(SINK_ENABLE_SEND_TSFILE_LIMIT, CONNECTOR_ENABLE_SEND_TSFILE_LIMIT),
+ CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE);
}
@Override
@@ -100,6 +113,14 @@ public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
return PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(fileName, position, payLoad);
}
+ @Override
+ protected void mayLimitRateAndRecordIO(final long requiredBytes) {
+ PipeResourceMetrics.getInstance().recordDiskIO(requiredBytes);
+ if (enableSendTsFileLimit) {
+ TsFileSendRateLimiter.getInstance().acquire(requiredBytes);
+ }
+ }
+
@Override
public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception {
// PipeProcessor can change the type of TabletInsertionEvent
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java
index f70e18c0651..f12a8125f91 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java
@@ -163,4 +163,9 @@ public class IoTDBSchemaRegionConnector extends IoTDBDataNodeSyncConnector {
final String fileName, final long position, final byte[] payLoad) throws IOException {
return PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq(fileName, position, payLoad);
}
+
+ @Override
+ protected void mayLimitRateAndRecordIO(final long requiredBytes) {
+ // Do nothing
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
index 8158c22d1a6..7fd7e37139e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
@@ -200,11 +200,13 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
SOURCE_HISTORY_ENABLE_KEY,
SOURCE_REALTIME_ENABLE_KEY)) {
LOGGER.warn(
- "When {}, {}, {} or {} is specified, specifying {}, {}, {} and {} is invalid.",
+ "When {}, {}, {} or {} is specified, specifying {}, {}, {}, {}, {} and {} is invalid.",
SOURCE_START_TIME_KEY,
EXTRACTOR_START_TIME_KEY,
SOURCE_END_TIME_KEY,
EXTRACTOR_END_TIME_KEY,
+ SOURCE_HISTORY_ENABLE_KEY,
+ EXTRACTOR_HISTORY_ENABLE_KEY,
SOURCE_HISTORY_START_TIME_KEY,
EXTRACTOR_HISTORY_START_TIME_KEY,
SOURCE_HISTORY_END_TIME_KEY,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
index 37f8eb5e26a..19a8060f89c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
@@ -26,7 +26,9 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Counter;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.MetricType;
@@ -40,8 +42,8 @@ public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_TS_FILE_USED_MEMORY = "PipeTsFileUsedMemory";
private static final String PIPE_TOTAL_MEMORY = "PipeTotalMemory";
-
private static final String PIPE_FLOATING_MEMORY = "PipeFloatingMemory";
+ private Counter diskIOCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
//////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
@@ -96,6 +98,10 @@ public class PipeResourceMetrics implements IMetricSet {
MetricLevel.IMPORTANT,
PipeDataNodeResourceManager.ref(),
PipePhantomReferenceManager::getPhantomReferenceCount);
+ // tsFile send rate
+ diskIOCounter =
+ metricService.getOrCreateCounter(
+ Metric.PIPE_TSFILE_SEND_DISK_IO.toString(), MetricLevel.IMPORTANT);
}
@Override
@@ -130,6 +136,12 @@ public class PipeResourceMetrics implements IMetricSet {
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_SIZE.toString());
// phantom reference count
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_PHANTOM_REFERENCE_COUNT.toString());
+
+ metricService.remove(MetricType.RATE, Metric.PIPE_TSFILE_SEND_DISK_IO.toString());
+ }
+
+ public void recordDiskIO(final long bytes) {
+ diskIOCounter.inc(bytes);
}
//////////////////////////// singleton ////////////////////////////
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java
index eaaf376a819..3be223287a2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java
@@ -19,78 +19,28 @@
package org.apache.iotdb.db.storageengine.load.limiter;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import org.apache.iotdb.commons.pipe.connector.limiter.GlobalRateLimiter;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet;
-import com.google.common.util.concurrent.AtomicDouble;
-import com.google.common.util.concurrent.RateLimiter;
-
-import java.util.concurrent.TimeUnit;
-
-public class LoadTsFileRateLimiter {
+public class LoadTsFileRateLimiter extends GlobalRateLimiter {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private final AtomicDouble throughputBytesPerSecond =
- new AtomicDouble(CONFIG.getLoadWriteThroughputBytesPerSecond());
- private final RateLimiter loadWriteRateLimiter;
-
+ @Override
public void acquire(long bytes) {
LoadTsFileCostMetricsSet.getInstance().recordDiskIO(bytes);
-
- if (reloadParams()) {
- return;
- }
-
- while (bytes > 0) {
- if (bytes > Integer.MAX_VALUE) {
- tryAcquireWithRateCheck(Integer.MAX_VALUE);
- bytes -= Integer.MAX_VALUE;
- } else {
- tryAcquireWithRateCheck((int) bytes);
- return;
- }
- }
+ super.acquire(bytes);
}
- private void tryAcquireWithRateCheck(final int bytes) {
- while (!loadWriteRateLimiter.tryAcquire(
- bytes,
- PipeConfig.getInstance().getRateLimiterHotReloadCheckIntervalMs(),
- TimeUnit.MILLISECONDS)) {
- if (reloadParams()) {
- return;
- }
- }
- }
-
- private boolean reloadParams() {
- final double throughputBytesPerSecondLimit = CONFIG.getLoadWriteThroughputBytesPerSecond();
-
- if (throughputBytesPerSecond.get() != throughputBytesPerSecondLimit) {
- throughputBytesPerSecond.set(throughputBytesPerSecondLimit);
- loadWriteRateLimiter.setRate(
- // if throughput <= 0, disable rate limiting
- throughputBytesPerSecondLimit <= 0 ? Double.MAX_VALUE : throughputBytesPerSecondLimit);
- }
-
- // For performance, we don't need to acquire rate limiter if throughput <= 0
- return throughputBytesPerSecondLimit <= 0;
+ @Override
+ protected double getThroughputBytesPerSecond() {
+ return CONFIG.getLoadWriteThroughputBytesPerSecond();
}
//////////////////////////// Singleton ////////////////////////////
- private LoadTsFileRateLimiter() {
- final double throughputBytesPerSecondLimit = throughputBytesPerSecond.get();
- loadWriteRateLimiter =
- // if throughput <= 0, disable rate limiting
- throughputBytesPerSecondLimit <= 0
- ? RateLimiter.create(Double.MAX_VALUE)
- : RateLimiter.create(throughputBytesPerSecondLimit);
- }
-
private static class LoadTsFileRateLimiterHolder {
private static final LoadTsFileRateLimiter INSTANCE = new LoadTsFileRateLimiter();
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index dfcbba78cff..9dfe5b68017 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -277,6 +277,7 @@ public class CommonConfig {
private int pipeAsyncConnectorMaxTsFileClientNumber =
Math.max(16, Runtime.getRuntime().availableProcessors());
+ private double pipeSendTsFileRateLimitBytesPerSecond = 32 * MB;
private double pipeAllSinksRateLimitBytesPerSecond = -1;
private int rateLimiterHotReloadCheckIntervalMs = 1000;
@@ -1921,6 +1922,21 @@ public class CommonConfig {
logger.info("pipeTransferTsFileSync is set to {}", pipeTransferTsFileSync);
}
+ public double getPipeSendTsFileRateLimitBytesPerSecond() {
+ return pipeSendTsFileRateLimitBytesPerSecond;
+ }
+
+ public void setPipeSendTsFileRateLimitBytesPerSecond(
+ double pipeSendTsFileRateLimitBytesPerSecond) {
+ if (this.pipeSendTsFileRateLimitBytesPerSecond == pipeSendTsFileRateLimitBytesPerSecond) {
+ return;
+ }
+ this.pipeSendTsFileRateLimitBytesPerSecond = pipeSendTsFileRateLimitBytesPerSecond;
+ logger.info(
+ "pipeSendTsFileRateLimitBytesPerSecond is set to {}",
+ pipeSendTsFileRateLimitBytesPerSecond);
+ }
+
public double getPipeAllSinksRateLimitBytesPerSecond() {
return pipeAllSinksRateLimitBytesPerSecond;
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
index 4ce2fd19093..620666ded48 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
@@ -217,6 +217,10 @@ public class PipeConfig {
return COMMON_CONFIG.getPipeAsyncConnectorMaxTsFileClientNumber();
}
+ public double getPipeSendTsFileRateLimitBytesPerSecond() {
+ return COMMON_CONFIG.getPipeSendTsFileRateLimitBytesPerSecond();
+ }
+
public double getPipeAllConnectorsRateLimitBytesPerSecond() {
return COMMON_CONFIG.getPipeAllSinksRateLimitBytesPerSecond();
}
@@ -537,6 +541,8 @@ public class PipeConfig {
"PipeAsyncConnectorMaxTsFileClientNumber: {}",
getPipeAsyncConnectorMaxTsFileClientNumber());
+ LOGGER.info(
+ "PipeSendTsFileRateLimitBytesPerSecond: {}", getPipeSendTsFileRateLimitBytesPerSecond());
LOGGER.info(
"PipeAllConnectorsRateLimitBytesPerSecond: {}",
getPipeAllConnectorsRateLimitBytesPerSecond());
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
index d6bad9d18f6..7822934478e 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
@@ -594,6 +594,12 @@ public class PipeDescriptor {
config.setPipeAsyncConnectorMaxTsFileClientNumber(Integer.parseInt(value));
}
+ value =
+ parserPipeConfig(properties, "pipe_send_tsfile_rate_limit_bytes_per_second", isHotModify);
+ if (value != null) {
+ config.setPipeSendTsFileRateLimitBytesPerSecond(Double.parseDouble(value));
+ }
+
value = parserPipeConfig(properties, "pipe_all_sinks_rate_limit_bytes_per_second", isHotModify);
if (value != null) {
config.setPipeAllSinksRateLimitBytesPerSecond(Double.parseDouble(value));
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
index c6780934703..ede91b27652 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
@@ -215,6 +215,11 @@ public class PipeConnectorConstant {
public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE = Zstd.minCompressionLevel();
public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE = Zstd.maxCompressionLevel();
+ public static final String CONNECTOR_ENABLE_SEND_TSFILE_LIMIT =
+ "connector.enable-send-tsfile-limit";
+ public static final String SINK_ENABLE_SEND_TSFILE_LIMIT = "sink.enable-send-tsfile-limit";
+ public static final boolean CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE = false;
+
public static final String CONNECTOR_RATE_LIMIT_KEY = "connector.rate-limit-bytes-per-second";
public static final String SINK_RATE_LIMIT_KEY = "sink.rate-limit-bytes-per-second";
public static final double CONNECTOR_RATE_LIMIT_DEFAULT_VALUE = -1;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRPCRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRPCRateLimiter.java
new file mode 100644
index 00000000000..ee450d16330
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRPCRateLimiter.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.connector.limiter;
+
+import org.apache.iotdb.commons.pipe.config.PipeConfig;
+
+/** This is a global rate limiter for all connectors. */
+public class GlobalRPCRateLimiter extends GlobalRateLimiter {
+
+ private static final PipeConfig CONFIG = PipeConfig.getInstance();
+
+ @Override
+ protected double getThroughputBytesPerSecond() {
+ return CONFIG.getPipeAllConnectorsRateLimitBytesPerSecond();
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java
index 08190eff500..bb583901796 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java
@@ -26,18 +26,17 @@ import com.google.common.util.concurrent.RateLimiter;
import java.util.concurrent.TimeUnit;
-/** This is a global rate limiter for all connectors. */
-public class GlobalRateLimiter {
-
- private static final PipeConfig CONFIG = PipeConfig.getInstance();
+public abstract class GlobalRateLimiter {
private final AtomicDouble throughputBytesPerSecond =
- new AtomicDouble(CONFIG.getPipeAllConnectorsRateLimitBytesPerSecond());
+ new AtomicDouble(getThroughputBytesPerSecond());
+
private final RateLimiter rateLimiter;
public GlobalRateLimiter() {
final double throughputBytesPerSecondLimit = throughputBytesPerSecond.get();
rateLimiter =
+ // if throughput <= 0, disable rate limiting
throughputBytesPerSecondLimit <= 0
? RateLimiter.create(Double.MAX_VALUE)
: RateLimiter.create(throughputBytesPerSecondLimit);
@@ -71,8 +70,7 @@ public class GlobalRateLimiter {
}
private boolean reloadParams() {
- final double throughputBytesPerSecondLimit =
- CONFIG.getPipeAllConnectorsRateLimitBytesPerSecond();
+ final double throughputBytesPerSecondLimit = getThroughputBytesPerSecond();
if (throughputBytesPerSecond.get() != throughputBytesPerSecondLimit) {
throughputBytesPerSecond.set(throughputBytesPerSecondLimit);
@@ -84,4 +82,6 @@ public class GlobalRateLimiter {
// For performance, we don't need to acquire rate limiter if throughput <= 0
return throughputBytesPerSecondLimit <= 0;
}
+
+ protected abstract double getThroughputBytesPerSecond();
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/TsFileSendRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/TsFileSendRateLimiter.java
new file mode 100644
index 00000000000..ff5e4e111b3
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/TsFileSendRateLimiter.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.connector.limiter;
+
+import org.apache.iotdb.commons.pipe.config.PipeConfig;
+
+public class TsFileSendRateLimiter extends GlobalRateLimiter {
+
+ private static final PipeConfig CONFIG = PipeConfig.getInstance();
+
+ @Override
+ protected double getThroughputBytesPerSecond() {
+ return CONFIG.getPipeSendTsFileRateLimitBytesPerSecond();
+ }
+
+ //////////////////////////// Singleton ////////////////////////////
+
+ private static class TsFileSendRateLimiterHolder {
+
+ private static final TsFileSendRateLimiter INSTANCE = new TsFileSendRateLimiter();
+
+ private TsFileSendRateLimiterHolder() {
+ // Prevent instantiation
+ }
+ }
+
+ public static TsFileSendRateLimiter getInstance() {
+ return TsFileSendRateLimiter.TsFileSendRateLimiterHolder.INSTANCE;
+ }
+}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java
index 01fcd45ffba..444ba634a81 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java
@@ -259,6 +259,7 @@ public abstract class IoTDBAirGapConnector extends IoTDBConnector {
long position = 0;
try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) {
while (true) {
+ mayLimitRateAndRecordIO(readFileBufferSize);
final int readLength = reader.read(readBuffer);
if (readLength == -1) {
break;
@@ -294,6 +295,8 @@ public abstract class IoTDBAirGapConnector extends IoTDBConnector {
}
}
+ protected abstract void mayLimitRateAndRecordIO(final long requiredBytes);
+
protected abstract boolean mayNeedHandshakeWhenFail();
protected abstract byte[] getTransferSingleFilePieceBytes(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java
index a7dacb49553..ab34a7e5463 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java
@@ -25,7 +25,7 @@ import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskConnectorRuntimeE
import org.apache.iotdb.commons.pipe.connector.compressor.PipeCompressor;
import org.apache.iotdb.commons.pipe.connector.compressor.PipeCompressorConfig;
import org.apache.iotdb.commons.pipe.connector.compressor.PipeCompressorFactory;
-import org.apache.iotdb.commons.pipe.connector.limiter.GlobalRateLimiter;
+import org.apache.iotdb.commons.pipe.connector.limiter.GlobalRPCRateLimiter;
import org.apache.iotdb.commons.pipe.connector.limiter.PipeEndPointRateLimiter;
import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferCompressedReq;
import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler;
@@ -165,7 +165,7 @@ public abstract class IoTDBConnector implements PipeConnector {
private static final Map<Pair<String, Long>, PipeEndPointRateLimiter>
PIPE_END_POINT_RATE_LIMITER_MAP = new ConcurrentHashMap<>();
private double endPointRateLimitBytesPerSecond = -1;
- private static final GlobalRateLimiter GLOBAL_RATE_LIMITER = new GlobalRateLimiter();
+ private static final GlobalRPCRateLimiter GLOBAL_RATE_LIMITER = new GlobalRPCRateLimiter();
protected boolean isTabletBatchModeEnabled = true;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java
index 51e532d737d..c0a94c5645d 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java
@@ -173,6 +173,7 @@ public abstract class IoTDBSslSyncConnector extends IoTDBConnector {
long position = 0;
try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) {
while (true) {
+ mayLimitRateAndRecordIO(readFileBufferSize);
final int readLength = reader.read(readBuffer);
if (readLength == -1) {
break;
@@ -242,6 +243,8 @@ public abstract class IoTDBSslSyncConnector extends IoTDBConnector {
protected abstract PipeTransferFilePieceReq getTransferMultiFilePieceReq(
final String fileName, final long position, final byte[] payLoad) throws IOException;
+ protected abstract void mayLimitRateAndRecordIO(final long requiredBytes);
+
@Override
public void close() {
if (clientManager != null) {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index cd91c788884..698ff13abe6 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -165,6 +165,7 @@ public enum Metric {
PIPE_LINKED_TSFILE_COUNT("pipe_linked_tsfile_count"),
PIPE_LINKED_TSFILE_SIZE("pipe_linked_tsfile_size"),
PIPE_PHANTOM_REFERENCE_COUNT("pipe_phantom_reference_count"),
+ PIPE_TSFILE_SEND_DISK_IO("pipe_tsfile_send_disk_io"),
PIPE_ASYNC_CONNECTOR_RETRY_EVENT_QUEUE_SIZE("pipe_async_connector_retry_event_queue_size"),
PIPE_EVENT_COMMIT_QUEUE_SIZE("pipe_event_commit_queue_size"),
PIPE_PROCEDURE("pipe_procedure"),
(iotdb) 04/11: Pipe: Fixed the ping-pong of certain realtime tsFiles (#15949)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 097f725a90118a5a62191b00ff8f658b85e798b6
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Tue Jul 15 23:56:33 2025 +0800
Pipe: Fixed the ping-pong of certain realtime tsFiles (#15949)
---
.../db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java
index 28660fd0e5d..c7990b24d45 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java
@@ -170,6 +170,7 @@ public class LoadTsFileDispatcherImpl implements IFragInstanceDispatcher {
final TsFileResource tsFileResource = ((LoadSingleTsFileNode) planNode).getTsFileResource();
try {
PipeDataNodeAgent.runtime().assignProgressIndexForTsFileLoad(tsFileResource);
+ tsFileResource.setGeneratedByPipe(isGeneratedByPipe);
tsFileResource.serialize();
StorageEngine.getInstance()
(iotdb) 06/11: Revert "Pipe: Added rate limiter for tsFile sending (#15765) (#15947)"
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1ee10537dda88b1e93436a1f2ea252541dc11886
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Jul 16 18:07:30 2025 +0800
Revert "Pipe: Added rate limiter for tsFile sending (#15765) (#15947)"
This reverts commit 025bf970cf138040b49b59464003e3643a53e984.
---
.../protocol/IoTDBConfigRegionAirGapConnector.java | 5 --
.../protocol/IoTDBConfigRegionConnector.java | 5 --
.../task/builder/PipeDataNodeTaskBuilder.java | 68 ++++++++--------------
.../airgap/IoTDBDataRegionAirGapConnector.java | 31 ----------
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 5 --
.../async/IoTDBDataRegionAsyncConnector.java | 14 -----
.../async/handler/PipeTransferTsFileHandler.java | 6 --
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 21 -------
.../thrift/sync/IoTDBSchemaRegionConnector.java | 5 --
.../dataregion/IoTDBDataRegionExtractor.java | 4 +-
.../pipe/metric/overview/PipeResourceMetrics.java | 14 +----
.../load/limiter/LoadTsFileRateLimiter.java | 64 +++++++++++++++++---
.../apache/iotdb/commons/conf/CommonConfig.java | 16 -----
.../iotdb/commons/pipe/config/PipeConfig.java | 6 --
.../iotdb/commons/pipe/config/PipeDescriptor.java | 6 --
.../config/constant/PipeConnectorConstant.java | 5 --
.../connector/limiter/GlobalRPCRateLimiter.java | 33 -----------
.../pipe/connector/limiter/GlobalRateLimiter.java | 14 ++---
.../connector/limiter/TsFileSendRateLimiter.java | 47 ---------------
.../connector/protocol/IoTDBAirGapConnector.java | 3 -
.../pipe/connector/protocol/IoTDBConnector.java | 4 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 3 -
.../iotdb/commons/service/metric/enums/Metric.java | 1 -
23 files changed, 91 insertions(+), 289 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java
index 595182d47d5..c5fdc4765f9 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionAirGapConnector.java
@@ -85,11 +85,6 @@ public class IoTDBConfigRegionAirGapConnector extends IoTDBAirGapConnector {
return PipeTransferConfigNodeHandshakeV2Req.toTPipeTransferBytes(params);
}
- @Override
- protected void mayLimitRateAndRecordIO(final long requiredBytes) {
- // Do nothing
- }
-
@Override
protected boolean mayNeedHandshakeWhenFail() {
return true;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java
index f04e18027b0..3761f696875 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java
@@ -97,11 +97,6 @@ public class IoTDBConfigRegionConnector extends IoTDBSslSyncConnector {
return PipeTransferConfigSnapshotPieceReq.toTPipeTransferReq(fileName, position, payLoad);
}
- @Override
- protected void mayLimitRateAndRecordIO(final long requiredBytes) {
- // Do nothing
- }
-
@Override
public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception {
throw new UnsupportedOperationException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
index 6ba1acae9bb..f84cb73fd69 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
@@ -52,10 +52,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstan
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_KEY;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY;
public class PipeDataNodeTaskBuilder {
@@ -173,6 +170,10 @@ public class PipeDataNodeTaskBuilder {
extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_SNAPSHOT_VALUE)
|| extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_QUERY_VALUE);
+ if (!insertionDeletionListeningOptionPair.right
+ && !shouldTerminatePipeOnAllHistoricalEventsConsumed) {
+ return;
+ }
} catch (final IllegalPathException e) {
LOGGER.warn(
"PipeDataNodeTaskBuilder failed to parse 'inclusion' and 'exclusion' parameters: {}",
@@ -181,52 +182,29 @@ public class PipeDataNodeTaskBuilder {
return;
}
- if (insertionDeletionListeningOptionPair.right
- || shouldTerminatePipeOnAllHistoricalEventsConsumed) {
- final Boolean isRealtime =
- connectorParameters.getBooleanByKeys(
- PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY,
- PipeConnectorConstant.SINK_REALTIME_FIRST_KEY);
- if (isRealtime == null) {
- connectorParameters.addAttribute(
- PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY, "false");
- if (insertionDeletionListeningOptionPair.right) {
- LOGGER.info(
- "PipeDataNodeTaskBuilder: When 'inclusion' contains 'data.delete', 'realtime-first' is defaulted to 'false' to prevent sync issues after deletion.");
- } else {
- LOGGER.info(
- "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' is defaulted to 'false' to prevent premature halt before transfer completion.");
- }
- } else if (isRealtime) {
- if (insertionDeletionListeningOptionPair.right) {
- LOGGER.warn(
- "PipeDataNodeTaskBuilder: When 'inclusion' includes 'data.delete', 'realtime-first' set to 'true' may result in data synchronization issues after deletion.");
- } else {
- LOGGER.warn(
- "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' set to 'true' may cause prevent premature halt before transfer completion.");
- }
+ final Boolean isRealtime =
+ connectorParameters.getBooleanByKeys(
+ PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY,
+ PipeConnectorConstant.SINK_REALTIME_FIRST_KEY);
+ if (isRealtime == null) {
+ connectorParameters.addAttribute(PipeConnectorConstant.CONNECTOR_REALTIME_FIRST_KEY, "false");
+ if (insertionDeletionListeningOptionPair.right) {
+ LOGGER.info(
+ "PipeDataNodeTaskBuilder: When 'inclusion' contains 'data.delete', 'realtime-first' is defaulted to 'false' to prevent sync issues after deletion.");
+ } else {
+ LOGGER.info(
+ "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' is defaulted to 'false' to prevent premature halt before transfer completion.");
}
+ return;
}
- final boolean isRealtimeEnabled =
- extractorParameters.getBooleanOrDefault(
- Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
- EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE);
-
- if (isRealtimeEnabled && !shouldTerminatePipeOnAllHistoricalEventsConsumed) {
- final Boolean enableSendTsFileLimit =
- connectorParameters.getBooleanByKeys(
- PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT,
- PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT);
-
- if (enableSendTsFileLimit == null) {
- connectorParameters.addAttribute(
- PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT, "true");
- LOGGER.info(
- "PipeDataNodeTaskBuilder: When the realtime sync is enabled, we enable rate limiter in sending tsfile by default to reserve disk and network IO for realtime sending.");
- } else if (!enableSendTsFileLimit) {
+ if (isRealtime) {
+ if (insertionDeletionListeningOptionPair.right) {
+ LOGGER.warn(
+ "PipeDataNodeTaskBuilder: When 'inclusion' includes 'data.delete', 'realtime-first' set to 'true' may result in data synchronization issues after deletion.");
+ } else {
LOGGER.warn(
- "PipeDataNodeTaskBuilder: When the realtime sync is enabled, not enabling the rate limiter in sending tsfile may introduce delay for realtime sending.");
+ "PipeDataNodeTaskBuilder: When extractor uses snapshot model, 'realtime-first' set to 'true' may cause prevent premature halt before transfer completion.");
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
index 57b92262abb..bc4805e8c8e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.pipe.connector.protocol.airgap;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.pipe.connector.limiter.TsFileSendRateLimiter;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletBinaryReq;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletInsertNodeReq;
@@ -35,12 +34,9 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertio
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
-import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
-import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
@@ -53,32 +49,13 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Objects;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT;
-
public class IoTDBDataRegionAirGapConnector extends IoTDBDataNodeAirGapConnector {
private static final Logger LOGGER =
LoggerFactory.getLogger(IoTDBDataRegionAirGapConnector.class);
- private boolean enableSendTsFileLimit;
-
- @Override
- public void customize(
- final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration)
- throws Exception {
- super.customize(parameters, configuration);
-
- enableSendTsFileLimit =
- parameters.getBooleanOrDefault(
- Arrays.asList(SINK_ENABLE_SEND_TSFILE_LIMIT, CONNECTOR_ENABLE_SEND_TSFILE_LIMIT),
- CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE);
- }
-
@Override
public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception {
// PipeProcessor can change the type of TabletInsertionEvent
@@ -316,14 +293,6 @@ public class IoTDBDataRegionAirGapConnector extends IoTDBDataNodeAirGapConnector
}
}
- @Override
- protected void mayLimitRateAndRecordIO(final long requiredBytes) {
- PipeResourceMetrics.getInstance().recordDiskIO(requiredBytes);
- if (enableSendTsFileLimit) {
- TsFileSendRateLimiter.getInstance().acquire(requiredBytes);
- }
- }
-
@Override
protected byte[] getTransferSingleFilePieceBytes(
final String fileName, final long position, final byte[] payLoad) throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java
index 84cc085509c..342807a5dfb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBSchemaRegionAirGapConnector.java
@@ -143,11 +143,6 @@ public class IoTDBSchemaRegionAirGapConnector extends IoTDBDataNodeAirGapConnect
}
}
- @Override
- protected void mayLimitRateAndRecordIO(final long requiredBytes) {
- // Do nothing
- }
-
@Override
protected byte[] getTransferSingleFilePieceBytes(
final String fileName, final long position, final byte[] payLoad) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
index f78a52a8281..0dacf0cb912 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
@@ -78,11 +78,8 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_LEADER_CACHE_ENABLE_KEY;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_SSL_ENABLE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PATH_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_SSL_TRUST_STORE_PWD_KEY;
@@ -118,8 +115,6 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
private final Map<PipeTransferTrackableHandler, PipeTransferTrackableHandler> pendingHandlers =
new ConcurrentHashMap<>();
- private boolean enableSendTsFileLimit;
-
@Override
public void validate(final PipeParameterValidator validator) throws Exception {
super.validate(validator);
@@ -175,11 +170,6 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
if (isTabletBatchModeEnabled) {
tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters);
}
-
- enableSendTsFileLimit =
- parameters.getBooleanOrDefault(
- Arrays.asList(SINK_ENABLE_SEND_TSFILE_LIMIT, CONNECTOR_ENABLE_SEND_TSFILE_LIMIT),
- CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE);
}
@Override
@@ -692,10 +682,6 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
events.forEach(this::addFailureEventToRetryQueue);
}
- public boolean isEnableSendTsFileLimit() {
- return enableSendTsFileLimit;
- }
-
//////////////////////////// Operations for close ////////////////////////////
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
index dbbf489b5a5..7353ea91e91 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.connector.protocol.thrift.async.handler;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.pipe.connector.limiter.TsFileSendRateLimiter;
import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.utils.RetryUtils;
@@ -33,7 +32,6 @@ import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransfer
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFileSealWithModReq;
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeTsFileMemoryBlock;
@@ -166,10 +164,6 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
client.setShouldReturnSelf(false);
client.setTimeoutDynamically(clientManager.getConnectionTimeout());
- PipeResourceMetrics.getInstance().recordDiskIO(readFileBufferSize);
- if (connector.isEnableSendTsFileLimit()) {
- TsFileSendRateLimiter.getInstance().acquire(readFileBufferSize);
- }
final int readLength = reader.read(readBuffer);
if (readLength == -1) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
index e0fd0b11827..d996cac2820 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.connector.protocol.thrift.sync;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.pipe.connector.client.IoTDBSyncClient;
-import org.apache.iotdb.commons.pipe.connector.limiter.TsFileSendRateLimiter;
import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferFilePieceReq;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.utils.RetryUtils;
@@ -44,7 +43,6 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertio
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
@@ -67,22 +65,16 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE;
-import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.SINK_ENABLE_SEND_TSFILE_LIMIT;
-
public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionSyncConnector.class);
private PipeTransferBatchReqBuilder tabletBatchBuilder;
- private boolean enableSendTsFileLimit;
@Override
public void customize(
@@ -94,11 +86,6 @@ public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
if (isTabletBatchModeEnabled) {
tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters);
}
-
- enableSendTsFileLimit =
- parameters.getBooleanOrDefault(
- Arrays.asList(SINK_ENABLE_SEND_TSFILE_LIMIT, CONNECTOR_ENABLE_SEND_TSFILE_LIMIT),
- CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE);
}
@Override
@@ -113,14 +100,6 @@ public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
return PipeTransferTsFilePieceWithModReq.toTPipeTransferReq(fileName, position, payLoad);
}
- @Override
- protected void mayLimitRateAndRecordIO(final long requiredBytes) {
- PipeResourceMetrics.getInstance().recordDiskIO(requiredBytes);
- if (enableSendTsFileLimit) {
- TsFileSendRateLimiter.getInstance().acquire(requiredBytes);
- }
- }
-
@Override
public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception {
// PipeProcessor can change the type of TabletInsertionEvent
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java
index f12a8125f91..f70e18c0651 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBSchemaRegionConnector.java
@@ -163,9 +163,4 @@ public class IoTDBSchemaRegionConnector extends IoTDBDataNodeSyncConnector {
final String fileName, final long position, final byte[] payLoad) throws IOException {
return PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq(fileName, position, payLoad);
}
-
- @Override
- protected void mayLimitRateAndRecordIO(final long requiredBytes) {
- // Do nothing
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
index 7fd7e37139e..8158c22d1a6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
@@ -200,13 +200,11 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
SOURCE_HISTORY_ENABLE_KEY,
SOURCE_REALTIME_ENABLE_KEY)) {
LOGGER.warn(
- "When {}, {}, {} or {} is specified, specifying {}, {}, {}, {}, {} and {} is invalid.",
+ "When {}, {}, {} or {} is specified, specifying {}, {}, {} and {} is invalid.",
SOURCE_START_TIME_KEY,
EXTRACTOR_START_TIME_KEY,
SOURCE_END_TIME_KEY,
EXTRACTOR_END_TIME_KEY,
- SOURCE_HISTORY_ENABLE_KEY,
- EXTRACTOR_HISTORY_ENABLE_KEY,
SOURCE_HISTORY_START_TIME_KEY,
EXTRACTOR_HISTORY_START_TIME_KEY,
SOURCE_HISTORY_END_TIME_KEY,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
index 19a8060f89c..37f8eb5e26a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
@@ -26,9 +26,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.metrics.AbstractMetricService;
-import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
-import org.apache.iotdb.metrics.type.Counter;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.MetricType;
@@ -42,8 +40,8 @@ public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_TS_FILE_USED_MEMORY = "PipeTsFileUsedMemory";
private static final String PIPE_TOTAL_MEMORY = "PipeTotalMemory";
+
private static final String PIPE_FLOATING_MEMORY = "PipeFloatingMemory";
- private Counter diskIOCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
//////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
@@ -98,10 +96,6 @@ public class PipeResourceMetrics implements IMetricSet {
MetricLevel.IMPORTANT,
PipeDataNodeResourceManager.ref(),
PipePhantomReferenceManager::getPhantomReferenceCount);
- // tsFile send rate
- diskIOCounter =
- metricService.getOrCreateCounter(
- Metric.PIPE_TSFILE_SEND_DISK_IO.toString(), MetricLevel.IMPORTANT);
}
@Override
@@ -136,12 +130,6 @@ public class PipeResourceMetrics implements IMetricSet {
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_SIZE.toString());
// phantom reference count
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_PHANTOM_REFERENCE_COUNT.toString());
-
- metricService.remove(MetricType.RATE, Metric.PIPE_TSFILE_SEND_DISK_IO.toString());
- }
-
- public void recordDiskIO(final long bytes) {
- diskIOCounter.inc(bytes);
}
//////////////////////////// singleton ////////////////////////////
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java
index 3be223287a2..eaaf376a819 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/limiter/LoadTsFileRateLimiter.java
@@ -19,28 +19,78 @@
package org.apache.iotdb.db.storageengine.load.limiter;
-import org.apache.iotdb.commons.pipe.connector.limiter.GlobalRateLimiter;
+import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet;
-public class LoadTsFileRateLimiter extends GlobalRateLimiter {
+import com.google.common.util.concurrent.AtomicDouble;
+import com.google.common.util.concurrent.RateLimiter;
+
+import java.util.concurrent.TimeUnit;
+
+public class LoadTsFileRateLimiter {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- @Override
+ private final AtomicDouble throughputBytesPerSecond =
+ new AtomicDouble(CONFIG.getLoadWriteThroughputBytesPerSecond());
+ private final RateLimiter loadWriteRateLimiter;
+
public void acquire(long bytes) {
LoadTsFileCostMetricsSet.getInstance().recordDiskIO(bytes);
- super.acquire(bytes);
+
+ if (reloadParams()) {
+ return;
+ }
+
+ while (bytes > 0) {
+ if (bytes > Integer.MAX_VALUE) {
+ tryAcquireWithRateCheck(Integer.MAX_VALUE);
+ bytes -= Integer.MAX_VALUE;
+ } else {
+ tryAcquireWithRateCheck((int) bytes);
+ return;
+ }
+ }
}
- @Override
- protected double getThroughputBytesPerSecond() {
- return CONFIG.getLoadWriteThroughputBytesPerSecond();
+ private void tryAcquireWithRateCheck(final int bytes) {
+ while (!loadWriteRateLimiter.tryAcquire(
+ bytes,
+ PipeConfig.getInstance().getRateLimiterHotReloadCheckIntervalMs(),
+ TimeUnit.MILLISECONDS)) {
+ if (reloadParams()) {
+ return;
+ }
+ }
+ }
+
+ private boolean reloadParams() {
+ final double throughputBytesPerSecondLimit = CONFIG.getLoadWriteThroughputBytesPerSecond();
+
+ if (throughputBytesPerSecond.get() != throughputBytesPerSecondLimit) {
+ throughputBytesPerSecond.set(throughputBytesPerSecondLimit);
+ loadWriteRateLimiter.setRate(
+ // if throughput <= 0, disable rate limiting
+ throughputBytesPerSecondLimit <= 0 ? Double.MAX_VALUE : throughputBytesPerSecondLimit);
+ }
+
+ // For performance, we don't need to acquire rate limiter if throughput <= 0
+ return throughputBytesPerSecondLimit <= 0;
}
//////////////////////////// Singleton ////////////////////////////
+ private LoadTsFileRateLimiter() {
+ final double throughputBytesPerSecondLimit = throughputBytesPerSecond.get();
+ loadWriteRateLimiter =
+ // if throughput <= 0, disable rate limiting
+ throughputBytesPerSecondLimit <= 0
+ ? RateLimiter.create(Double.MAX_VALUE)
+ : RateLimiter.create(throughputBytesPerSecondLimit);
+ }
+
private static class LoadTsFileRateLimiterHolder {
private static final LoadTsFileRateLimiter INSTANCE = new LoadTsFileRateLimiter();
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index 9dfe5b68017..dfcbba78cff 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -277,7 +277,6 @@ public class CommonConfig {
private int pipeAsyncConnectorMaxTsFileClientNumber =
Math.max(16, Runtime.getRuntime().availableProcessors());
- private double pipeSendTsFileRateLimitBytesPerSecond = 32 * MB;
private double pipeAllSinksRateLimitBytesPerSecond = -1;
private int rateLimiterHotReloadCheckIntervalMs = 1000;
@@ -1922,21 +1921,6 @@ public class CommonConfig {
logger.info("pipeTransferTsFileSync is set to {}", pipeTransferTsFileSync);
}
- public double getPipeSendTsFileRateLimitBytesPerSecond() {
- return pipeSendTsFileRateLimitBytesPerSecond;
- }
-
- public void setPipeSendTsFileRateLimitBytesPerSecond(
- double pipeSendTsFileRateLimitBytesPerSecond) {
- if (this.pipeSendTsFileRateLimitBytesPerSecond == pipeSendTsFileRateLimitBytesPerSecond) {
- return;
- }
- this.pipeSendTsFileRateLimitBytesPerSecond = pipeSendTsFileRateLimitBytesPerSecond;
- logger.info(
- "pipeSendTsFileRateLimitBytesPerSecond is set to {}",
- pipeSendTsFileRateLimitBytesPerSecond);
- }
-
public double getPipeAllSinksRateLimitBytesPerSecond() {
return pipeAllSinksRateLimitBytesPerSecond;
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
index 620666ded48..4ce2fd19093 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
@@ -217,10 +217,6 @@ public class PipeConfig {
return COMMON_CONFIG.getPipeAsyncConnectorMaxTsFileClientNumber();
}
- public double getPipeSendTsFileRateLimitBytesPerSecond() {
- return COMMON_CONFIG.getPipeSendTsFileRateLimitBytesPerSecond();
- }
-
public double getPipeAllConnectorsRateLimitBytesPerSecond() {
return COMMON_CONFIG.getPipeAllSinksRateLimitBytesPerSecond();
}
@@ -541,8 +537,6 @@ public class PipeConfig {
"PipeAsyncConnectorMaxTsFileClientNumber: {}",
getPipeAsyncConnectorMaxTsFileClientNumber());
- LOGGER.info(
- "PipeSendTsFileRateLimitBytesPerSecond: {}", getPipeSendTsFileRateLimitBytesPerSecond());
LOGGER.info(
"PipeAllConnectorsRateLimitBytesPerSecond: {}",
getPipeAllConnectorsRateLimitBytesPerSecond());
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
index 7822934478e..d6bad9d18f6 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
@@ -594,12 +594,6 @@ public class PipeDescriptor {
config.setPipeAsyncConnectorMaxTsFileClientNumber(Integer.parseInt(value));
}
- value =
- parserPipeConfig(properties, "pipe_send_tsfile_rate_limit_bytes_per_second", isHotModify);
- if (value != null) {
- config.setPipeSendTsFileRateLimitBytesPerSecond(Double.parseDouble(value));
- }
-
value = parserPipeConfig(properties, "pipe_all_sinks_rate_limit_bytes_per_second", isHotModify);
if (value != null) {
config.setPipeAllSinksRateLimitBytesPerSecond(Double.parseDouble(value));
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
index ede91b27652..c6780934703 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeConnectorConstant.java
@@ -215,11 +215,6 @@ public class PipeConnectorConstant {
public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MIN_VALUE = Zstd.minCompressionLevel();
public static final int CONNECTOR_COMPRESSOR_ZSTD_LEVEL_MAX_VALUE = Zstd.maxCompressionLevel();
- public static final String CONNECTOR_ENABLE_SEND_TSFILE_LIMIT =
- "connector.enable-send-tsfile-limit";
- public static final String SINK_ENABLE_SEND_TSFILE_LIMIT = "sink.enable-send-tsfile-limit";
- public static final boolean CONNECTOR_ENABLE_SEND_TSFILE_LIMIT_DEFAULT_VALUE = false;
-
public static final String CONNECTOR_RATE_LIMIT_KEY = "connector.rate-limit-bytes-per-second";
public static final String SINK_RATE_LIMIT_KEY = "sink.rate-limit-bytes-per-second";
public static final double CONNECTOR_RATE_LIMIT_DEFAULT_VALUE = -1;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRPCRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRPCRateLimiter.java
deleted file mode 100644
index ee450d16330..00000000000
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRPCRateLimiter.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.commons.pipe.connector.limiter;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-
-/** This is a global rate limiter for all connectors. */
-public class GlobalRPCRateLimiter extends GlobalRateLimiter {
-
- private static final PipeConfig CONFIG = PipeConfig.getInstance();
-
- @Override
- protected double getThroughputBytesPerSecond() {
- return CONFIG.getPipeAllConnectorsRateLimitBytesPerSecond();
- }
-}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java
index bb583901796..08190eff500 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java
@@ -26,17 +26,18 @@ import com.google.common.util.concurrent.RateLimiter;
import java.util.concurrent.TimeUnit;
-public abstract class GlobalRateLimiter {
+/** This is a global rate limiter for all connectors. */
+public class GlobalRateLimiter {
- private final AtomicDouble throughputBytesPerSecond =
- new AtomicDouble(getThroughputBytesPerSecond());
+ private static final PipeConfig CONFIG = PipeConfig.getInstance();
+ private final AtomicDouble throughputBytesPerSecond =
+ new AtomicDouble(CONFIG.getPipeAllConnectorsRateLimitBytesPerSecond());
private final RateLimiter rateLimiter;
public GlobalRateLimiter() {
final double throughputBytesPerSecondLimit = throughputBytesPerSecond.get();
rateLimiter =
- // if throughput <= 0, disable rate limiting
throughputBytesPerSecondLimit <= 0
? RateLimiter.create(Double.MAX_VALUE)
: RateLimiter.create(throughputBytesPerSecondLimit);
@@ -70,7 +71,8 @@ public abstract class GlobalRateLimiter {
}
private boolean reloadParams() {
- final double throughputBytesPerSecondLimit = getThroughputBytesPerSecond();
+ final double throughputBytesPerSecondLimit =
+ CONFIG.getPipeAllConnectorsRateLimitBytesPerSecond();
if (throughputBytesPerSecond.get() != throughputBytesPerSecondLimit) {
throughputBytesPerSecond.set(throughputBytesPerSecondLimit);
@@ -82,6 +84,4 @@ public abstract class GlobalRateLimiter {
// For performance, we don't need to acquire rate limiter if throughput <= 0
return throughputBytesPerSecondLimit <= 0;
}
-
- protected abstract double getThroughputBytesPerSecond();
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/TsFileSendRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/TsFileSendRateLimiter.java
deleted file mode 100644
index ff5e4e111b3..00000000000
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/TsFileSendRateLimiter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.commons.pipe.connector.limiter;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-
-public class TsFileSendRateLimiter extends GlobalRateLimiter {
-
- private static final PipeConfig CONFIG = PipeConfig.getInstance();
-
- @Override
- protected double getThroughputBytesPerSecond() {
- return CONFIG.getPipeSendTsFileRateLimitBytesPerSecond();
- }
-
- //////////////////////////// Singleton ////////////////////////////
-
- private static class TsFileSendRateLimiterHolder {
-
- private static final TsFileSendRateLimiter INSTANCE = new TsFileSendRateLimiter();
-
- private TsFileSendRateLimiterHolder() {
- // Prevent instantiation
- }
- }
-
- public static TsFileSendRateLimiter getInstance() {
- return TsFileSendRateLimiter.TsFileSendRateLimiterHolder.INSTANCE;
- }
-}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java
index 444ba634a81..01fcd45ffba 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBAirGapConnector.java
@@ -259,7 +259,6 @@ public abstract class IoTDBAirGapConnector extends IoTDBConnector {
long position = 0;
try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) {
while (true) {
- mayLimitRateAndRecordIO(readFileBufferSize);
final int readLength = reader.read(readBuffer);
if (readLength == -1) {
break;
@@ -295,8 +294,6 @@ public abstract class IoTDBAirGapConnector extends IoTDBConnector {
}
}
- protected abstract void mayLimitRateAndRecordIO(final long requiredBytes);
-
protected abstract boolean mayNeedHandshakeWhenFail();
protected abstract byte[] getTransferSingleFilePieceBytes(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java
index ab34a7e5463..a7dacb49553 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java
@@ -25,7 +25,7 @@ import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskConnectorRuntimeE
import org.apache.iotdb.commons.pipe.connector.compressor.PipeCompressor;
import org.apache.iotdb.commons.pipe.connector.compressor.PipeCompressorConfig;
import org.apache.iotdb.commons.pipe.connector.compressor.PipeCompressorFactory;
-import org.apache.iotdb.commons.pipe.connector.limiter.GlobalRPCRateLimiter;
+import org.apache.iotdb.commons.pipe.connector.limiter.GlobalRateLimiter;
import org.apache.iotdb.commons.pipe.connector.limiter.PipeEndPointRateLimiter;
import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferCompressedReq;
import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler;
@@ -165,7 +165,7 @@ public abstract class IoTDBConnector implements PipeConnector {
private static final Map<Pair<String, Long>, PipeEndPointRateLimiter>
PIPE_END_POINT_RATE_LIMITER_MAP = new ConcurrentHashMap<>();
private double endPointRateLimitBytesPerSecond = -1;
- private static final GlobalRPCRateLimiter GLOBAL_RATE_LIMITER = new GlobalRPCRateLimiter();
+ private static final GlobalRateLimiter GLOBAL_RATE_LIMITER = new GlobalRateLimiter();
protected boolean isTabletBatchModeEnabled = true;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java
index c0a94c5645d..51e532d737d 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java
@@ -173,7 +173,6 @@ public abstract class IoTDBSslSyncConnector extends IoTDBConnector {
long position = 0;
try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) {
while (true) {
- mayLimitRateAndRecordIO(readFileBufferSize);
final int readLength = reader.read(readBuffer);
if (readLength == -1) {
break;
@@ -243,8 +242,6 @@ public abstract class IoTDBSslSyncConnector extends IoTDBConnector {
protected abstract PipeTransferFilePieceReq getTransferMultiFilePieceReq(
final String fileName, final long position, final byte[] payLoad) throws IOException;
- protected abstract void mayLimitRateAndRecordIO(final long requiredBytes);
-
@Override
public void close() {
if (clientManager != null) {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index 698ff13abe6..cd91c788884 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -165,7 +165,6 @@ public enum Metric {
PIPE_LINKED_TSFILE_COUNT("pipe_linked_tsfile_count"),
PIPE_LINKED_TSFILE_SIZE("pipe_linked_tsfile_size"),
PIPE_PHANTOM_REFERENCE_COUNT("pipe_phantom_reference_count"),
- PIPE_TSFILE_SEND_DISK_IO("pipe_tsfile_send_disk_io"),
PIPE_ASYNC_CONNECTOR_RETRY_EVENT_QUEUE_SIZE("pipe_async_connector_retry_event_queue_size"),
PIPE_EVENT_COMMIT_QUEUE_SIZE("pipe_event_commit_queue_size"),
PIPE_PROCEDURE("pipe_procedure"),
(iotdb) 08/11: [to dev/1.3] Set global time filter before analyzing last query
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7e403f4f5d6ddd01b4d47af1a87d7c57cc1adf67
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Thu Jul 24 21:06:01 2025 +0800
[to dev/1.3] Set global time filter before analyzing last query
---
.../org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index afc964ea301..1c8983d045f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -299,6 +299,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
analyzeGlobalTimeFilter(analysis, queryStatement);
if (queryStatement.isLastQuery()) {
+ context.generateGlobalTimeFilter(analysis);
return analyzeLastQuery(queryStatement, analysis, schemaTree, context);
}
(iotdb) 09/11: [To dev/1.3] Optimize partition cache getRegionReplicaSet interface performance by batching
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 968d65cba0404f6dcff490508221eb45d6a84bc9
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Fri Jul 25 14:16:35 2025 +0800
[To dev/1.3] Optimize partition cache getRegionReplicaSet interface performance by batching
---
.../plan/analyze/ClusterPartitionFetcher.java | 54 +++-
.../analyze/cache/partition/PartitionCache.java | 347 ++++++++++++---------
.../plan/analyze/cache/PartitionCacheTest.java | 5 +-
3 files changed, 242 insertions(+), 164 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java
index e76973c516b..ff19dd79cf5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java
@@ -56,9 +56,10 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -415,15 +416,21 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
private SchemaPartition parseSchemaPartitionTableResp(
TSchemaPartitionTableResp schemaPartitionTableResp) {
Map<String, Map<TSeriesPartitionSlot, TRegionReplicaSet>> regionReplicaMap = new HashMap<>();
- for (Map.Entry<String, Map<TSeriesPartitionSlot, TConsensusGroupId>> entry1 :
+ for (final Map.Entry<String, Map<TSeriesPartitionSlot, TConsensusGroupId>> entry1 :
schemaPartitionTableResp.getSchemaPartitionTable().entrySet()) {
- Map<TSeriesPartitionSlot, TRegionReplicaSet> result1 =
- regionReplicaMap.computeIfAbsent(entry1.getKey(), k -> new HashMap<>());
- for (Map.Entry<TSeriesPartitionSlot, TConsensusGroupId> entry2 :
- entry1.getValue().entrySet()) {
- TSeriesPartitionSlot seriesPartitionSlot = entry2.getKey();
- TConsensusGroupId consensusGroupId = entry2.getValue();
- result1.put(seriesPartitionSlot, partitionCache.getRegionReplicaSet(consensusGroupId));
+ String database = entry1.getKey();
+ final Map<TSeriesPartitionSlot, TRegionReplicaSet> result1 =
+ regionReplicaMap.computeIfAbsent(database, k -> new HashMap<>());
+
+ Map<TSeriesPartitionSlot, TConsensusGroupId> orderedMap =
+ new LinkedHashMap<>(entry1.getValue());
+ List<TConsensusGroupId> orderedGroupIds = new ArrayList<>(orderedMap.values());
+ List<TRegionReplicaSet> regionReplicaSets =
+ partitionCache.getRegionReplicaSet(orderedGroupIds);
+
+ int index = 0;
+ for (Map.Entry<TSeriesPartitionSlot, TConsensusGroupId> entry2 : orderedMap.entrySet()) {
+ result1.put(entry2.getKey(), regionReplicaSets.get(index++));
}
}
@@ -443,6 +450,29 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
}
private DataPartition parseDataPartitionResp(TDataPartitionTableResp dataPartitionTableResp) {
+ final Set<TConsensusGroupId> uniqueConsensusGroupIds = new HashSet<>();
+ for (final Map<
+ String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TConsensusGroupId>>>>
+ partitionTable : Collections.singleton(dataPartitionTableResp.getDataPartitionTable())) {
+ for (final Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TConsensusGroupId>>>
+ seriesPartitionMap : partitionTable.values()) {
+ for (final Map<TTimePartitionSlot, List<TConsensusGroupId>> timePartitionMap :
+ seriesPartitionMap.values()) {
+ for (final List<TConsensusGroupId> consensusGroupIds : timePartitionMap.values()) {
+ uniqueConsensusGroupIds.addAll(consensusGroupIds);
+ }
+ }
+ }
+ }
+
+ final List<TRegionReplicaSet> allRegionReplicaSets =
+ partitionCache.getRegionReplicaSet(new ArrayList<>(uniqueConsensusGroupIds));
+ final List<TConsensusGroupId> consensusGroupIds = new ArrayList<>(uniqueConsensusGroupIds);
+ final Map<TConsensusGroupId, TRegionReplicaSet> regionReplicaSetMap = new HashMap<>();
+ for (int i = 0; i < allRegionReplicaSets.size(); i++) {
+ regionReplicaSetMap.put(consensusGroupIds.get(i), allRegionReplicaSets.get(i));
+ }
+
Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>>
regionReplicaSet = new HashMap<>();
for (Map.Entry<
@@ -456,9 +486,9 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
result1.computeIfAbsent(entry2.getKey(), k -> new HashMap<>());
for (Map.Entry<TTimePartitionSlot, List<TConsensusGroupId>> entry3 :
entry2.getValue().entrySet()) {
- List<TRegionReplicaSet> regionReplicaSets = new LinkedList<>();
- for (TConsensusGroupId consensusGroupId : entry3.getValue()) {
- regionReplicaSets.add(partitionCache.getRegionReplicaSet(consensusGroupId));
+ final List<TRegionReplicaSet> regionReplicaSets = new ArrayList<>();
+ for (TConsensusGroupId groupId : entry3.getValue()) {
+ regionReplicaSets.add(regionReplicaSetMap.get(groupId));
}
result2.put(entry3.getKey(), regionReplicaSets);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java
index e649822bac4..8c701bcd126 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java
@@ -68,9 +68,9 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -317,10 +317,12 @@ public class PartitionCache {
for (String devicePath : devicePaths) {
String storageGroupName = getStorageGroupName(devicePath);
if (null == storageGroupName) {
- logger.debug(
- "[{} Cache] miss when search device {}",
- CacheMetrics.STORAGE_GROUP_CACHE_NAME,
- devicePath);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search device {}",
+ CacheMetrics.STORAGE_GROUP_CACHE_NAME,
+ devicePath);
+ }
status = false;
if (failFast) {
break;
@@ -335,10 +337,12 @@ public class PartitionCache {
if (!status) {
result.setFailed();
}
- logger.debug(
- "[{} Cache] hit when search device {}",
- CacheMetrics.STORAGE_GROUP_CACHE_NAME,
- devicePaths);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] hit when search device {}",
+ CacheMetrics.STORAGE_GROUP_CACHE_NAME,
+ devicePaths);
+ }
cacheMetrics.record(status, CacheMetrics.STORAGE_GROUP_CACHE_NAME);
} finally {
storageGroupCacheLock.readLock().unlock();
@@ -439,26 +443,30 @@ public class PartitionCache {
/**
* get regionReplicaSet from local and confignode
*
- * @param consensusGroupId the id of consensus group
- * @return regionReplicaSet
+ * @param consensusGroupIds the ids of consensus group
+ * @return List<regionReplicaSet>
* @throws RuntimeException if failed to get regionReplicaSet from confignode
* @throws StatementAnalyzeException if there are exception when try to get latestRegionRouteMap
*/
- public TRegionReplicaSet getRegionReplicaSet(TConsensusGroupId consensusGroupId) {
- TRegionReplicaSet result;
+ public List<TRegionReplicaSet> getRegionReplicaSet(List<TConsensusGroupId> consensusGroupIds) {
+ if (consensusGroupIds.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<TRegionReplicaSet> result;
// try to get regionReplicaSet from cache
regionReplicaSetLock.readLock().lock();
try {
- result = groupIdToReplicaSetMap.get(consensusGroupId);
+ result = getRegionReplicaSetInternal(consensusGroupIds);
} finally {
regionReplicaSetLock.readLock().unlock();
}
- if (result == null) {
+ if (result.isEmpty()) {
// if not hit then try to get regionReplicaSet from confignode
regionReplicaSetLock.writeLock().lock();
try {
- // verify that there are not hit in cache
- if (!groupIdToReplicaSetMap.containsKey(consensusGroupId)) {
+ // double check after getting the write lock
+ result = getRegionReplicaSetInternal(consensusGroupIds);
+ if (result.isEmpty()) {
try (ConfigNodeClient client =
configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
TRegionRouteMapResp resp = client.getLatestRegionRouteMap();
@@ -470,18 +478,18 @@ public class PartitionCache {
resp.getStatus(),
resp.getRegionRouteMap());
}
+ result = getRegionReplicaSetInternal(consensusGroupIds);
// if confignode don't have then will throw RuntimeException
- if (!groupIdToReplicaSetMap.containsKey(consensusGroupId)) {
+ if (result.isEmpty()) {
// failed to get RegionReplicaSet from confignode
throw new RuntimeException(
- "Failed to get replicaSet of consensus group[id= " + consensusGroupId + "]");
+ "Failed to get replicaSet of consensus groups[ids= " + consensusGroupIds + "]");
}
} catch (ClientManagerException | TException e) {
throw new StatementAnalyzeException(
"An error occurred when executing getRegionReplicaSet():" + e.getMessage());
}
}
- result = groupIdToReplicaSetMap.get(consensusGroupId);
} finally {
regionReplicaSetLock.writeLock().unlock();
}
@@ -490,6 +498,20 @@ public class PartitionCache {
return result;
}
+ private List<TRegionReplicaSet> getRegionReplicaSetInternal(
+ List<TConsensusGroupId> consensusGroupIds) {
+ List<TRegionReplicaSet> result = new ArrayList<>(consensusGroupIds.size());
+ for (TConsensusGroupId groupId : consensusGroupIds) {
+ TRegionReplicaSet replicaSet = groupIdToReplicaSetMap.get(groupId);
+ if (replicaSet != null) {
+ result.add(replicaSet);
+ } else {
+ return Collections.emptyList();
+ }
+ }
+ return result;
+ }
+
/**
* update regionReplicaSetMap according to timestamp
*
@@ -551,34 +573,45 @@ public class PartitionCache {
schemaPartitionCache.getIfPresent(storageGroupName);
if (null == schemaPartitionTable) {
// if database not find, then return cache miss.
- logger.debug(
- "[{} Cache] miss when search database {}",
- CacheMetrics.SCHEMA_PARTITION_CACHE_NAME,
- storageGroupName);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search database {}",
+ CacheMetrics.SCHEMA_PARTITION_CACHE_NAME,
+ storageGroupName);
+ }
cacheMetrics.record(false, CacheMetrics.SCHEMA_PARTITION_CACHE_NAME);
return null;
}
Map<TSeriesPartitionSlot, TConsensusGroupId> map =
schemaPartitionTable.getSchemaPartitionMap();
// check cache for each device
+ List<TSeriesPartitionSlot> seriesPartitionSlots = new ArrayList<>(entry.getValue().size());
+ List<TConsensusGroupId> consensusGroupIds = new ArrayList<>(entry.getValue().size());
for (String device : entry.getValue()) {
TSeriesPartitionSlot seriesPartitionSlot =
partitionExecutor.getSeriesPartitionSlot(device);
if (!map.containsKey(seriesPartitionSlot)) {
// if one device not find, then return cache miss.
- logger.debug(
- "[{} Cache] miss when search device {}",
- CacheMetrics.SCHEMA_PARTITION_CACHE_NAME,
- device);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search device {}",
+ CacheMetrics.SCHEMA_PARTITION_CACHE_NAME,
+ device);
+ }
cacheMetrics.record(false, CacheMetrics.SCHEMA_PARTITION_CACHE_NAME);
return null;
}
- TConsensusGroupId consensusGroupId = map.get(seriesPartitionSlot);
- TRegionReplicaSet regionReplicaSet = getRegionReplicaSet(consensusGroupId);
- regionReplicaSetMap.put(seriesPartitionSlot, regionReplicaSet);
+ seriesPartitionSlots.add(seriesPartitionSlot);
+ consensusGroupIds.add(map.get(seriesPartitionSlot));
+ }
+ List<TRegionReplicaSet> replicaSets = getRegionReplicaSet(consensusGroupIds);
+ for (int i = 0; i < replicaSets.size(); i++) {
+ regionReplicaSetMap.put(seriesPartitionSlots.get(i), replicaSets.get(i));
}
}
- logger.debug("[{} Cache] hit", CacheMetrics.SCHEMA_PARTITION_CACHE_NAME);
+ if (logger.isDebugEnabled()) {
+ logger.debug("[{} Cache] hit", CacheMetrics.SCHEMA_PARTITION_CACHE_NAME);
+ }
// cache hit
cacheMetrics.record(true, CacheMetrics.SCHEMA_PARTITION_CACHE_NAME);
return new SchemaPartition(
@@ -656,22 +689,115 @@ public class PartitionCache {
cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME);
return null;
}
- Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>>
- dataPartitionMap = new HashMap<>();
+
+ final Set<TConsensusGroupId> allConsensusGroupIds = new HashSet<>();
+ final Map<TConsensusGroupId, HashSet<TimeSlotRegionInfo>> consensusGroupToTimeSlotMap =
+ new HashMap<>();
+
// check cache for each database
for (Map.Entry<String, List<DataPartitionQueryParam>> entry :
storageGroupToQueryParamsMap.entrySet()) {
- if (null == entry.getValue()
- || entry.getValue().isEmpty()
- || !getStorageGroupDataPartition(dataPartitionMap, entry.getKey(), entry.getValue())) {
+ String databaseName = entry.getKey();
+ List<DataPartitionQueryParam> params = entry.getValue();
+
+ if (null == params || params.isEmpty()) {
cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME);
return null;
}
+
+ DataPartitionTable dataPartitionTable = dataPartitionCache.getIfPresent(databaseName);
+ if (null == dataPartitionTable) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search database {}",
+ CacheMetrics.DATA_PARTITION_CACHE_NAME,
+ databaseName);
+ }
+ cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME);
+ return null;
+ }
+
+ Map<TSeriesPartitionSlot, SeriesPartitionTable> cachedDatabasePartitionMap =
+ dataPartitionTable.getDataPartitionMap();
+
+ for (DataPartitionQueryParam param : params) {
+ TSeriesPartitionSlot seriesPartitionSlot;
+ if (null != param.getDevicePath()) {
+ seriesPartitionSlot = partitionExecutor.getSeriesPartitionSlot(param.getDevicePath());
+ } else {
+ return null;
+ }
+
+ SeriesPartitionTable cachedSeriesPartitionTable =
+ cachedDatabasePartitionMap.get(seriesPartitionSlot);
+ if (null == cachedSeriesPartitionTable) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search device {}",
+ CacheMetrics.DATA_PARTITION_CACHE_NAME,
+ param.getDevicePath());
+ }
+ cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME);
+ return null;
+ }
+
+ Map<TTimePartitionSlot, List<TConsensusGroupId>> cachedTimePartitionSlot =
+ cachedSeriesPartitionTable.getSeriesPartitionMap();
+
+ if (param.getTimePartitionSlotList().isEmpty()) {
+ return null;
+ }
+
+ for (TTimePartitionSlot timePartitionSlot : param.getTimePartitionSlotList()) {
+ List<TConsensusGroupId> cacheConsensusGroupIds =
+ cachedTimePartitionSlot.get(timePartitionSlot);
+ if (null == cacheConsensusGroupIds
+ || cacheConsensusGroupIds.isEmpty()
+ || null == timePartitionSlot) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search time partition {}",
+ CacheMetrics.DATA_PARTITION_CACHE_NAME,
+ timePartitionSlot);
+ }
+ cacheMetrics.record(false, CacheMetrics.DATA_PARTITION_CACHE_NAME);
+ return null;
+ }
+
+ for (TConsensusGroupId groupId : cacheConsensusGroupIds) {
+ allConsensusGroupIds.add(groupId);
+ consensusGroupToTimeSlotMap
+ .computeIfAbsent(groupId, k -> new HashSet<>())
+ .add(
+ new TimeSlotRegionInfo(databaseName, seriesPartitionSlot, timePartitionSlot));
+ }
+ }
+ }
}
+
+ final List<TConsensusGroupId> consensusGroupIds = new ArrayList<>(allConsensusGroupIds);
+ final List<TRegionReplicaSet> allRegionReplicaSets = getRegionReplicaSet(consensusGroupIds);
+
+ Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>>
+ dataPartitionMap = new HashMap<>();
+
+ for (int i = 0; i < allRegionReplicaSets.size(); i++) {
+ TConsensusGroupId groupId = consensusGroupIds.get(i);
+ TRegionReplicaSet replicaSet = allRegionReplicaSets.get(i);
+
+ for (TimeSlotRegionInfo info : consensusGroupToTimeSlotMap.get(groupId)) {
+ dataPartitionMap
+ .computeIfAbsent(info.databaseName, k -> new HashMap<>())
+ .computeIfAbsent(info.seriesPartitionSlot, k -> new HashMap<>())
+ .computeIfAbsent(info.timePartitionSlot, k -> new ArrayList<>())
+ .add(replicaSet);
+ }
+ }
+
if (logger.isDebugEnabled()) {
logger.debug("[{} Cache] hit", CacheMetrics.DATA_PARTITION_CACHE_NAME);
}
- // cache hit
+
cacheMetrics.record(true, CacheMetrics.DATA_PARTITION_CACHE_NAME);
return new DataPartition(dataPartitionMap, seriesSlotExecutorName, seriesPartitionSlotNum);
} finally {
@@ -679,124 +805,39 @@ public class PartitionCache {
}
}
- /**
- * get dataPartition from database
- *
- * @param dataPartitionMap result
- * @param storageGroupName database that need to get
- * @param dataPartitionQueryParams specific query params of data partition
- * @return whether hit
- */
- private boolean getStorageGroupDataPartition(
- Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>>
- dataPartitionMap,
- String storageGroupName,
- List<DataPartitionQueryParam> dataPartitionQueryParams) {
- DataPartitionTable dataPartitionTable = dataPartitionCache.getIfPresent(storageGroupName);
- if (null == dataPartitionTable) {
- if (logger.isDebugEnabled()) {
- logger.debug(
- "[{} Cache] miss when search database {}",
- CacheMetrics.DATA_PARTITION_CACHE_NAME,
- storageGroupName);
- }
- return false;
+ private static class TimeSlotRegionInfo {
+ final String databaseName;
+ final TSeriesPartitionSlot seriesPartitionSlot;
+ final TTimePartitionSlot timePartitionSlot;
+
+ TimeSlotRegionInfo(
+ String databaseName,
+ TSeriesPartitionSlot seriesPartitionSlot,
+ TTimePartitionSlot timePartitionSlot) {
+ this.databaseName = databaseName;
+ this.seriesPartitionSlot = seriesPartitionSlot;
+ this.timePartitionSlot = timePartitionSlot;
}
- Map<TSeriesPartitionSlot, SeriesPartitionTable> cachedStorageGroupPartitionMap =
- dataPartitionTable.getDataPartitionMap();
- Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>
- seriesSlotToTimePartitionMap =
- dataPartitionMap.computeIfAbsent(storageGroupName, k -> new HashMap<>());
- // check cache for each device
- for (DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) {
- if (!getDeviceDataPartition(
- seriesSlotToTimePartitionMap, dataPartitionQueryParam, cachedStorageGroupPartitionMap)) {
- return false;
- }
- }
- return true;
- }
- /**
- * get dataPartition from device
- *
- * @param seriesSlotToTimePartitionMap result
- * @param dataPartitionQueryParam specific query param of data partition
- * @param cachedStorageGroupPartitionMap all cached data partition map of related database
- * @return whether hit
- */
- private boolean getDeviceDataPartition(
- Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>
- seriesSlotToTimePartitionMap,
- DataPartitionQueryParam dataPartitionQueryParam,
- Map<TSeriesPartitionSlot, SeriesPartitionTable> cachedStorageGroupPartitionMap) {
- TSeriesPartitionSlot seriesPartitionSlot;
- if (null != dataPartitionQueryParam.getDevicePath()) {
- seriesPartitionSlot =
- partitionExecutor.getSeriesPartitionSlot(dataPartitionQueryParam.getDevicePath());
- } else {
- return false;
- }
- SeriesPartitionTable cachedSeriesPartitionTable =
- cachedStorageGroupPartitionMap.get(seriesPartitionSlot);
- if (null == cachedSeriesPartitionTable) {
- if (logger.isDebugEnabled()) {
- logger.debug(
- "[{} Cache] miss when search device {}",
- CacheMetrics.DATA_PARTITION_CACHE_NAME,
- dataPartitionQueryParam.getDevicePath());
- }
- return false;
- }
- Map<TTimePartitionSlot, List<TConsensusGroupId>> cachedTimePartitionSlot =
- cachedSeriesPartitionTable.getSeriesPartitionMap();
- Map<TTimePartitionSlot, List<TRegionReplicaSet>> timePartitionSlotListMap =
- seriesSlotToTimePartitionMap.computeIfAbsent(seriesPartitionSlot, k -> new HashMap<>());
- // Notice: when query all time partition, then miss
- if (dataPartitionQueryParam.getTimePartitionSlotList().isEmpty()) {
- return false;
- }
- // check cache for each time partition
- for (TTimePartitionSlot timePartitionSlot :
- dataPartitionQueryParam.getTimePartitionSlotList()) {
- if (!getTimeSlotDataPartition(
- timePartitionSlotListMap, timePartitionSlot, cachedTimePartitionSlot)) {
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
return false;
}
- }
- return true;
- }
- /**
- * get dataPartition from time slot
- *
- * @param timePartitionSlotListMap result
- * @param timePartitionSlot the specific time partition slot of data partition
- * @param cachedTimePartitionSlot all cached time slot map of related device
- * @return whether hit
- */
- private boolean getTimeSlotDataPartition(
- Map<TTimePartitionSlot, List<TRegionReplicaSet>> timePartitionSlotListMap,
- TTimePartitionSlot timePartitionSlot,
- Map<TTimePartitionSlot, List<TConsensusGroupId>> cachedTimePartitionSlot) {
- List<TConsensusGroupId> cacheConsensusGroupId = cachedTimePartitionSlot.get(timePartitionSlot);
- if (null == cacheConsensusGroupId
- || cacheConsensusGroupId.isEmpty()
- || null == timePartitionSlot) {
- if (logger.isDebugEnabled()) {
- logger.debug(
- "[{} Cache] miss when search time partition {}",
- CacheMetrics.DATA_PARTITION_CACHE_NAME,
- timePartitionSlot);
- }
- return false;
+ TimeSlotRegionInfo that = (TimeSlotRegionInfo) o;
+ return Objects.equals(databaseName, that.databaseName)
+ && Objects.equals(seriesPartitionSlot, that.seriesPartitionSlot)
+ && Objects.equals(timePartitionSlot, that.timePartitionSlot);
}
- List<TRegionReplicaSet> regionReplicaSets = new LinkedList<>();
- for (TConsensusGroupId consensusGroupId : cacheConsensusGroupId) {
- regionReplicaSets.add(getRegionReplicaSet(consensusGroupId));
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hashCode(databaseName);
+ result = 31 * result + Objects.hashCode(seriesPartitionSlot);
+ result = 31 * result + Objects.hashCode(timePartitionSlot);
+ return result;
}
- timePartitionSlotListMap.put(timePartitionSlot, regionReplicaSets);
- return true;
}
/**
@@ -879,12 +920,16 @@ public class PartitionCache {
// endregion
public void invalidAllCache() {
- logger.debug("[Partition Cache] invalid");
+ if (logger.isDebugEnabled()) {
+ logger.debug("[Partition Cache] invalid");
+ }
removeFromStorageGroupCache();
invalidAllDataPartitionCache();
invalidAllSchemaPartitionCache();
invalidReplicaSetCache();
- logger.debug("[Partition Cache] is invalid:{}", this);
+ if (logger.isDebugEnabled()) {
+ logger.debug("[Partition Cache] is invalid:{}", this);
+ }
}
@Override
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java
index ca9fcb062a7..43b3a867472 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java
@@ -258,7 +258,10 @@ public class PartitionCacheTest {
private void checkRegionReplicaSet(TConsensusGroupId consensusGroupId) {
try {
- assertNotNull(partitionCache.getRegionReplicaSet(consensusGroupId));
+ List<TRegionReplicaSet> regionReplicaSets =
+ partitionCache.getRegionReplicaSet(Collections.singletonList(consensusGroupId));
+ assertEquals(1, regionReplicaSets.size());
+ assertNotNull(regionReplicaSets.get(0));
} catch (Exception e) {
fail(e.getMessage());
}
(iotdb) 07/11: [To dev/1.3] Pipe: Added progressIndex reset function to the resource resetting script (#15957)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ebdf275a3509be03471abd73c5473bb88b87169a
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Wed Jul 16 19:00:21 2025 +0800
[To dev/1.3] Pipe: Added progressIndex reset function to the resource resetting script (#15957)
* Pipe: Added progressIndex reset function to the resource resetting script
* bat-fix
---
...pipe.bat => reset-resource-pipe-statistics.bat} | 4 +-
...y-pipe.sh => reset-resource-pipe-statistics.sh} | 4 +-
...va => TsFileResourcePipeStatisticsSetTool.java} | 125 ++++++++++++++-------
3 files changed, 86 insertions(+), 47 deletions(-)
diff --git a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.bat b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/reset-resource-pipe-statistics.bat
similarity index 92%
rename from iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.bat
rename to iotdb-core/datanode/src/assembly/resources/tools/tsfile/reset-resource-pipe-statistics.bat
index 7d2b867bba9..098072a4b4f 100644
--- a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.bat
+++ b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/reset-resource-pipe-statistics.bat
@@ -19,7 +19,7 @@
@echo off
echo ````````````````````````````````````````````````````````````````````````
-echo Starting Validating the isGeneratedByPipe Mark in TsFile Resources
+echo Starting Resetting the Pipe Related Statistics in TsFile Resources
echo ````````````````````````````````````````````````````````````````````````
if "%OS%" == "Windows_NT" setlocal
@@ -28,7 +28,7 @@ pushd %~dp0..\..
if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
popd
-if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.validate.TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.validate.TsFileResourcePipeStatisticsSetTool
if NOT DEFINED JAVA_HOME goto :err
@REM -----------------------------------------------------------------------------
diff --git a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.sh b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/reset-resource-pipe-statistics.sh
similarity index 90%
rename from iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.sh
rename to iotdb-core/datanode/src/assembly/resources/tools/tsfile/reset-resource-pipe-statistics.sh
index daabae671f3..5dc3fab8699 100644
--- a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/mark-is-generated-by-pipe.sh
+++ b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/reset-resource-pipe-statistics.sh
@@ -19,7 +19,7 @@
#
echo ------------------------------------------------------------------------------------
-echo Starting Validating the isGeneratedByPipe Mark in TsFile Resources
+echo Starting Resetting the Pipe Related Statistics in TsFile Resources
echo ------------------------------------------------------------------------------------
source "$(dirname "$0")/../../sbin/iotdb-common.sh"
@@ -45,7 +45,7 @@ for f in ${IOTDB_HOME}/lib/*.jar; do
CLASSPATH=${CLASSPATH}":"$f
done
-MAIN_CLASS=org.apache.iotdb.db.tools.validate.TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool
+MAIN_CLASS=org.apache.iotdb.db.tools.validate.TsFileResourcePipeStatisticsSetTool
"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
exit $?
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourcePipeStatisticsSetTool.java
similarity index 60%
rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool.java
rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourcePipeStatisticsSetTool.java
index e14f7e81d05..9c4c73361a7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileResourcePipeStatisticsSetTool.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.tools.validate;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.common.constant.TsFileConstant;
@@ -37,33 +38,34 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-public class TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool {
+public class TsFileResourcePipeStatisticsSetTool {
private static final Logger LOGGER =
- org.slf4j.LoggerFactory.getLogger(
- TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool.class);
+ org.slf4j.LoggerFactory.getLogger(TsFileResourcePipeStatisticsSetTool.class);
private static final String USAGE =
- "Usage: --expected true|false --dirs <dir1> <dir2> ...\n"
- + " --expected: whether the TsFileResource is expected to be generated by pipe\n"
+ "Usage: [--isGeneratedByPipe true|false] [--resetProgressIndex] --dirs <dir1> <dir2> ...\n"
+ + " --isGeneratedByPipe: whether the TsFileResource is isGeneratedByPipe to be generated by pipe\n"
+ + " --resetProgressIndex: whether to reset the TsFileResources' progressIndexes\n"
+ " --dirs: list of data directories to validate and repair";
private static final Set<File> dataDirs = new ConcurrentSkipListSet<>();
- private static final AtomicBoolean expectedMark = new AtomicBoolean(true);
+ private static AtomicBoolean isGeneratedByPipeMark = null;
+ private static boolean resetProgressIndex = false;
private static final AtomicLong runtime = new AtomicLong(System.currentTimeMillis());
private static final AtomicInteger totalTsFileNum = new AtomicInteger(0);
- private static final AtomicInteger toRepairTsFileNum = new AtomicInteger(0);
+ private static final AtomicInteger toResetFlagNum = new AtomicInteger(0);
+ private static final AtomicInteger toResetProgressIndexNum = new AtomicInteger(0);
+ private static final AtomicInteger changedNum = new AtomicInteger(0);
- // Usage: --expected true|false --dirs <dir1> <dir2> ...
+ // Usage: [--isGeneratedByPipe true|false] [--resetProgressIndex] --dirs <dir1> <dir2> ...
public static void main(String[] args) throws IOException {
parseCommandLineArgs(args);
final List<File> partitionDirs = findAllPartitionDirs();
partitionDirs.parallelStream()
- .forEach(
- TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool
- ::validateAndRepairTsFileResourcesInPartition);
+ .forEach(TsFileResourcePipeStatisticsSetTool::validateAndRepairTsFileResourcesInPartition);
printStatistics();
}
@@ -74,14 +76,17 @@ public class TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool {
if (args.length == 0
|| argSet.contains("--help")
|| argSet.contains("-h")
- || !(argSet.contains("--expected") && argSet.contains("--dirs"))) {
+ || !((argSet.contains("--isGeneratedByPipe") || argSet.contains("--resetProgressIndex"))
+ && argSet.contains("--dirs"))) {
LOGGER.info(USAGE);
System.exit(1);
}
for (int i = 0; i < args.length; i++) {
- if ("--expected".equals(args[i]) && i + 1 < args.length) {
- expectedMark.set(Boolean.parseBoolean(args[++i]));
+ if ("--isGeneratedByPipe".equals(args[i]) && i + 1 < args.length) {
+ isGeneratedByPipeMark = new AtomicBoolean(Boolean.parseBoolean(args[++i]));
+ } else if ("--resetProgressIndex".equals(args[i])) {
+ resetProgressIndex = true;
} else if ("--dirs".equals(args[i]) && i + 1 < args.length) {
i++;
while (i < args.length && !args[i].startsWith("--")) {
@@ -102,7 +107,8 @@ public class TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool {
}
LOGGER.info("------------------------------------------------------");
- LOGGER.info("Expected mark: {}", expectedMark.get());
+ LOGGER.info("isGeneratedByPipe mark: {}", isGeneratedByPipeMark);
+ LOGGER.info("resetProgressIndex: {}", resetProgressIndex);
LOGGER.info("Data directories: ");
for (File dir : dataDirs) {
LOGGER.info(" {}", dir.getAbsolutePath());
@@ -144,19 +150,20 @@ public class TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool {
}
private static void validateAndRepairTsFileResourcesInPartition(final File partitionDir) {
- final AtomicInteger totalResources = new AtomicInteger();
- final AtomicInteger toRepairResources = new AtomicInteger();
+ final AtomicInteger totalTsFileResource = new AtomicInteger(0);
+ final AtomicInteger toResetFlagResource = new AtomicInteger(0);
+ final AtomicInteger toResetProgressIndexResource = new AtomicInteger(0);
+ final AtomicInteger changedResource = new AtomicInteger(0);
try {
final List<TsFileResource> resources =
loadAllTsFileResources(Collections.singletonList(partitionDir));
- totalResources.addAndGet(resources.size());
+ totalTsFileResource.addAndGet(resources.size());
for (final TsFileResource resource : resources) {
try {
- if (validateAndRepairSingleTsFileResource(resource)) {
- toRepairResources.incrementAndGet();
- }
+ validateAndRepairSingleTsFileResource(
+ resource, toResetFlagResource, toResetProgressIndexResource, changedResource);
} catch (final Exception e) {
// Continue processing other resources even if one fails
LOGGER.warn(
@@ -174,13 +181,17 @@ public class TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool {
e);
}
- totalTsFileNum.addAndGet(totalResources.get());
- toRepairTsFileNum.addAndGet(toRepairResources.get());
+ totalTsFileNum.addAndGet(totalTsFileResource.get());
+ toResetFlagNum.addAndGet(toResetFlagResource.get());
+ toResetProgressIndexNum.addAndGet(toResetProgressIndexResource.get());
+ changedNum.addAndGet(changedResource.get());
LOGGER.info(
- "TimePartition {} has {} total resources, {} to repair resources. Process completed.",
+ "TimePartition {} has {} total resources, {} to set isGeneratedByPipe resources, {} to reset progressIndex resources, {} changed resources. Process completed.",
partitionDir,
- totalResources.get(),
- toRepairResources.get());
+ totalTsFileResource.get(),
+ toResetFlagResource.get(),
+ toResetProgressIndexResource.get(),
+ changedResource.get());
}
private static List<TsFileResource> loadAllTsFileResources(List<File> timePartitionDirs)
@@ -217,37 +228,63 @@ public class TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool {
* @param resource the TsFileResource to validate and repair
* @return true if the resource needs to be repaired and false if it is valid
*/
- private static boolean validateAndRepairSingleTsFileResource(TsFileResource resource) {
- if (resource.isGeneratedByPipe() == expectedMark.get()) {
+ private static void validateAndRepairSingleTsFileResource(
+ final TsFileResource resource,
+ final AtomicInteger toResetFlagResource,
+ final AtomicInteger toResetProgressIndexResource,
+ final AtomicInteger changedResource) {
+ boolean skip = true;
+ if (Objects.nonNull(isGeneratedByPipeMark)
+ && resource.isGeneratedByPipe() != isGeneratedByPipeMark.get()) {
// The resource is valid, no need to repair
- return false;
+ LOGGER.info(
+ "Repairing TsFileResource: {}, isGeneratedByPipe mark: {}, actual mark: {}",
+ resource.getTsFile().getAbsolutePath(),
+ isGeneratedByPipeMark.get(),
+ resource.isGeneratedByPipe());
+
+ toResetFlagResource.getAndIncrement();
+ skip = false;
}
- LOGGER.info(
- "Repairing TsFileResource: {}, expected mark: {}, actual mark: {}",
- resource.getTsFile().getAbsolutePath(),
- expectedMark.get(),
- resource.isGeneratedByPipe());
+ if (resetProgressIndex && resource.getMaxProgressIndex() != MinimumProgressIndex.INSTANCE) {
+ // The resource is valid, no need to repair
+ LOGGER.info(
+ "Resetting TsFileResource:{} 's progressIndex to minimum, original progressIndex: {}",
+ resource.getTsFile().getAbsolutePath(),
+ resource.getMaxProgressIndex());
+
+ toResetProgressIndexResource.getAndIncrement();
+ skip = false;
+ }
+ if (skip) {
+ return;
+ }
+ changedResource.getAndIncrement();
try {
repairSingleTsFileResource(resource);
LOGGER.info(
"Marked TsFileResource as {} in resource: {}",
- expectedMark.get(),
+ isGeneratedByPipeMark.get(),
+ resource.getTsFile().getAbsolutePath());
+ LOGGER.info(
+ "Reset TsFileResource:{} 's progressIndex to minimum.",
resource.getTsFile().getAbsolutePath());
} catch (final Exception e) {
LOGGER.warn(
- "ERROR: Failed to repair TsFileResource: {}, error: {}",
- resource.getTsFile().getAbsolutePath(),
- e.getMessage());
+ "ERROR: Failed to repair TsFileResource: {}", resource.getTsFile().getAbsolutePath(), e);
}
-
- return true;
}
private static void repairSingleTsFileResource(TsFileResource resource) throws IOException {
- resource.setGeneratedByPipe(expectedMark.get());
+ if (Objects.nonNull(isGeneratedByPipeMark)) {
+ resource.setGeneratedByPipe(isGeneratedByPipeMark.get());
+ }
+ if (resetProgressIndex) {
+ resource.setProgressIndex(MinimumProgressIndex.INSTANCE);
+ }
resource.serialize();
}
@@ -255,9 +292,11 @@ public class TsFileResourceIsGeneratedByPipeMarkValidationAndRepairTool {
LOGGER.info("------------------------------------------------------");
LOGGER.info("Validation and repair completed. Statistics:");
LOGGER.info(
- "Total time taken: {} ms, total TsFile resources: {}, repaired TsFile resources: {}",
+ "Total time taken: {} ms, total TsFile resources: {}, set isGeneratedByPipe resources: {}, reset progressIndex resources: {}, changed resources: {}",
System.currentTimeMillis() - runtime.get(),
totalTsFileNum.get(),
- toRepairTsFileNum.get());
+ toResetFlagNum.get(),
+ toResetProgressIndexNum.get(),
+ changedNum.get());
}
}
(iotdb) 10/11: [To dev/1.3] Pipe: Fixed the issue that frequent logins on the receiving end caused indicator leaks #16076 (#16085)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/1.3.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9da931acd609a0cbcf3fc78094507deaa4258f9d
Author: Zhenyu Luo <zh...@apache.org>
AuthorDate: Mon Aug 4 10:30:56 2025 +0800
[To dev/1.3] Pipe: Fixed the issue that frequent logins on the receiving end caused indicator leaks #16076 (#16085)
* Pipe: Fixed the issue that frequent logins on the receiving end caused indicator leaks
* fix
* spotless
* spotless
* spotless
* spotless
---
.../protocol/thrift/IoTDBDataNodeReceiver.java | 24 ++++++++++++++--------
1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index 9c841200ec2..4070773b8aa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -756,15 +756,21 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
@Override
protected TSStatus login() {
- final BasicOpenSessionResp openSessionResp =
- SESSION_MANAGER.login(
- SESSION_MANAGER.getCurrSession(),
- username,
- password,
- ZoneId.systemDefault().toString(),
- SessionManager.CURRENT_RPC_VERSION,
- IoTDBConstant.ClientVersion.V_1_0);
- return RpcUtils.getStatus(openSessionResp.getCode(), openSessionResp.getMessage());
+ final IClientSession session = SESSION_MANAGER.getCurrSession();
+
+ if (session != null && !session.isLogin()) {
+ final BasicOpenSessionResp openSessionResp =
+ SESSION_MANAGER.login(
+ session,
+ username,
+ password,
+ ZoneId.systemDefault().toString(),
+ SessionManager.CURRENT_RPC_VERSION,
+ IoTDBConstant.ClientVersion.V_1_0);
+ return RpcUtils.getStatus(openSessionResp.getCode(), openSessionResp.message);
+ }
+
+ return AuthorityChecker.checkUser(username, password);
}
@Override