You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ju...@apache.org on 2025/07/03 03:08:37 UTC
(iotdb) branch squash-dbgw updated: Comprehensive event (#15858)
This is an automated email from the ASF dual-hosted git repository.
justinchen pushed a commit to branch squash-dbgw
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/squash-dbgw by this push:
new 08aa3cce7a8 Comprehensive event (#15858)
08aa3cce7a8 is described below
commit 08aa3cce7a8f7f21941de1554754ce47bd94b54a
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Thu Jul 3 11:08:32 2025 +0800
Comprehensive event (#15858)
As the title said.
---
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 6 +-
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeClusterIT.java | 6 +-
.../IoTDBPipeConnectorCompressionIT.java | 7 +-
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 1 -
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 6 +-
.../pipe/it/autocreate/IoTDBPipeWithLoadIT.java | 6 +-
.../pipe/it/manual/AbstractPipeDualManualIT.java | 6 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 1 +
.../it/cluster/IoTDBSubscriptionRestartIT.java | 3 +-
.../it/dual/AbstractSubscriptionDualIT.java | 3 +
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 3 +
.../it/dual/IoTDBSubscriptionTimePrecisionIT.java | 2 +
.../it/dual/IoTDBSubscriptionTopicIT.java | 3 +
.../it/local/AbstractSubscriptionLocalIT.java | 7 +-
.../it/triple/AbstractSubscriptionTripleIT.java | 4 +
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 6 +-
.../event/dml/insertion/TsFileInsertionEvent.java | 10 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 8 +
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../connector/PipeConfigNodeThriftRequestTest.java | 4 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 30 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 -
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 585 ++++++++++----------
.../connector/PipeConnectorSubtaskLifeCycle.java | 5 +
.../PipeRealtimePriorityBlockingQueue.java | 154 +++++-
.../subtask/processor/PipeProcessorSubtask.java | 4 +-
.../evolvable/batch/PipeTabletEventBatch.java | 36 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 3 +-
.../evolvable/request/PipeTransferPlanNodeReq.java | 1 -
.../request/PipeTransferTabletBatchReq.java | 1 -
.../request/PipeTransferTabletBinaryReq.java | 1 -
.../request/PipeTransferTabletInsertNodeReq.java | 1 -
.../request/PipeTransferTabletRawReq.java | 1 -
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +-
.../PipeConsensusTransferBatchReqBuilder.java | 3 +-
.../async/IoTDBDataRegionAsyncConnector.java | 3 +-
.../PipeTransferTabletInsertionEventHandler.java | 2 +-
.../async/handler/PipeTransferTsFileHandler.java | 4 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 3 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 39 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 125 ++---
.../common/tablet/PipeRawTabletInsertionEvent.java | 6 +-
.../event/common/terminate/PipeTerminateEvent.java | 44 --
.../tsfile/PipeCompactedTsFileInsertionEvent.java | 252 +++++++++
.../common/tsfile/PipeTsFileInsertionEvent.java | 132 ++---
.../TsFileInsertionDataContainerProvider.java | 7 +-
.../db/pipe/event/realtime/PipeRealtimeEvent.java | 8 +-
.../event/realtime/PipeRealtimeEventFactory.java | 16 +-
.../dataregion/IoTDBDataRegionExtractor.java | 107 ++--
.../PipeHistoricalDataRegionTsFileExtractor.java | 143 ++---
.../realtime/PipeRealtimeDataRegionExtractor.java | 92 +++-
.../PipeRealtimeDataRegionHeartbeatExtractor.java | 28 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 191 ++-----
.../PipeRealtimeDataRegionLogExtractor.java | 66 +--
.../PipeRealtimeDataRegionTsFileExtractor.java | 60 +--
.../realtime/assigner/PipeDataRegionAssigner.java | 214 +++-----
.../listener/PipeInsertionDataNodeListener.java | 17 +-
.../matcher/CachedSchemaPatternMatcher.java | 202 ++-----
.../realtime/matcher/PipeDataRegionMatcher.java | 52 --
.../schemaregion/IoTDBSchemaRegionExtractor.java | 4 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 12 +-
.../PipeDataNodeRemainingEventAndTimeOperator.java | 38 +-
...ics.java => PipeDataNodeSinglePipeMetrics.java} | 119 +++--
.../pipe/metric/overview/PipeResourceMetrics.java | 39 +-
.../overview/PipeTsFileToTabletsMetrics.java | 3 +-
.../overview/PipeWALInsertNodeCacheMetrics.java | 86 ---
.../db/pipe/metric/source/PipeAssignerMetrics.java | 165 ------
.../twostage/exchange/payload/CombineRequest.java | 1 -
.../payload/FetchCombineResultRequest.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 16 +
...aNodeHardlinkOrCopiedFileDirStartupCleaner.java | 8 +-
.../pipe/resource/PipeDataNodeResourceManager.java | 25 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 33 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 12 +-
.../resource/tsfile/PipeCompactionManager.java | 69 +++
...ileResource.java => PipeTsFileMemResource.java} | 156 +-----
.../pipe/resource/tsfile/PipeTsFileResource.java | 248 +--------
.../resource/tsfile/PipeTsFileResourceManager.java | 310 +++++------
.../db/pipe/resource/wal/PipeWALResource.java | 169 ------
.../pipe/resource/wal/PipeWALResourceManager.java | 142 -----
.../wal/hardlink/PipeWALHardlinkResource.java | 47 --
.../hardlink/PipeWALHardlinkResourceManager.java | 152 ------
.../wal/selfhost/PipeWALSelfHostResource.java | 41 --
.../selfhost/PipeWALSelfHostResourceManager.java | 38 --
.../impl/DataNodeInternalRPCServiceImpl.java | 95 +++-
.../execution/config/sys/pipe/ShowPipeTask.java | 4 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 27 +
.../iotdb/db/storageengine/StorageEngine.java | 8 -
.../db/storageengine/dataregion/DataRegion.java | 2 +-
.../task/InsertionCrossSpaceCompactionTask.java | 4 +-
.../compaction/execute/utils/CompactionUtils.java | 4 +-
.../dataregion/memtable/TsFileProcessor.java | 32 +-
.../dataregion/tsfile/TsFileManager.java | 10 +
.../dataregion/tsfile/TsFileResource.java | 8 -
.../dataregion/wal/buffer/WALBuffer.java | 6 -
.../dataregion/wal/buffer/WALEntry.java | 4 +-
.../wal/checkpoint/CheckpointManager.java | 70 +--
.../dataregion/wal/checkpoint/MemTableInfo.java | 19 +-
.../dataregion/wal/io/WALByteBufReader.java | 13 -
.../dataregion/wal/node/WALFakeNode.java | 4 +-
.../storageengine/dataregion/wal/node/WALNode.java | 96 +---
.../dataregion/wal/utils/WALEntryHandler.java | 233 --------
.../dataregion/wal/utils/WALEntryPosition.java | 229 --------
.../dataregion/wal/utils/WALInsertNodeCache.java | 335 ------------
.../wal/utils/listener/WALFlushListener.java | 12 +-
...eeStatementDataTypeConvertExecutionVisitor.java | 169 ++----
.../converter/LoadTsFileDataTypeConverter.java | 42 +-
.../connector/PipeDataNodeThriftRequestTest.java | 12 -
.../event/TsFileInsertionDataContainerTest.java | 16 +-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 103 ++--
.../pattern/CachedSchemaPatternMatcherTest.java | 218 --------
.../resource/PipeTsFileResourceManagerTest.java | 96 ++--
.../PipeWALHardlinkResourceManagerTest.java | 114 ----
.../TsFileResourceProgressIndexTest.java | 23 +-
.../wal/checkpoint/CheckpointManagerTest.java | 172 ------
.../dataregion/wal/node/WALEntryHandlerTest.java | 586 ---------------------
.../dataregion/wal/node/WALNodeTest.java | 17 -
.../wal/node/WalDeleteOutdatedNewTest.java | 65 ---
.../wal/utils/WALInsertNodeCacheTest.java | 190 -------
.../iotdb/commons/concurrent/ThreadName.java | 8 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 404 +++++---------
.../commons/consensus/index/ProgressIndex.java | 2 +-
.../commons/consensus/index/ProgressIndexType.java | 8 +-
.../consensus/index/impl/IoTProgressIndex.java | 27 +-
.../consensus/index/impl/SegmentProgressIndex.java | 193 +++++++
.../commons/pipe/agent/task/PipeTaskAgent.java | 50 +-
.../task/connection/BlockingPendingQueue.java | 20 +-
.../connection/UnboundedBlockingPendingQueue.java | 2 +-
.../pipe/agent/task/meta/PipeRuntimeMeta.java | 9 +
.../commons/pipe/agent/task/meta/PipeTaskMeta.java | 25 +-
.../task/progress/PipeEventCommitManager.java | 25 +-
.../agent/task/progress/PipeEventCommitter.java | 5 +
.../iotdb/commons/pipe/config/PipeConfig.java | 144 ++---
.../iotdb/commons/pipe/config/PipeDescriptor.java | 133 ++---
.../config/constant/PipeConnectorConstant.java | 2 +-
.../thrift/request/PipeTransferFilePieceReq.java | 1 -
.../thrift/request/PipeTransferFileSealReqV1.java | 1 -
.../thrift/request/PipeTransferFileSealReqV2.java | 1 -
.../thrift/request/PipeTransferHandshakeV1Req.java | 1 -
.../thrift/request/PipeTransferHandshakeV2Req.java | 1 -
.../thrift/request/PipeTransferSliceReq.java | 1 -
.../iotdb/commons/pipe/event/EnrichedEvent.java | 13 +
.../iotdb/commons/service/metric/enums/Metric.java | 5 +-
.../org/apache/iotdb/commons/utils/FileUtils.java | 17 +
158 files changed, 2672 insertions(+), 6228 deletions(-)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
index fe4190d7708..a7d8918d1c9 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
@@ -429,6 +429,12 @@ public class MppCommonConfig extends MppBaseConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ setProperty("pipe_enable_memory_checked", String.valueOf(isPipeEnableMemoryCheck));
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
setProperty("pipe_air_gap_receiver_enabled", String.valueOf(isPipeAirGapReceiverEnabled));
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
index c2ade6eace0..e4a4ace7a44 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
@@ -438,6 +438,13 @@ public class MppSharedCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ dnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
+ cnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
dnConfig.setPipeAirGapReceiverEnabled(isPipeAirGapReceiverEnabled);
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
index 581c5f4345e..23a8dad317e 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
@@ -308,6 +308,11 @@ public class RemoteCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
return this;
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
index 16d50954209..178f182624b 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
@@ -138,6 +138,8 @@ public interface CommonConfig {
CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode);
+ CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck);
+
CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled);
CommonConfig setDriverTaskExecutionTimeSliceInMs(long driverTaskExecutionTimeSliceInMs);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
index 1817efdaa1d..59478fc36e6 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java
@@ -48,13 +48,15 @@ abstract class AbstractPipeDualAutoIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
index 65eb2458df7..9da5591bb2f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java
@@ -59,14 +59,16 @@ public class IoTDBPipeAutoConflictIT extends AbstractPipeDualAutoIT {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java
index d2ca8ce138c..c8d5bc0b471 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java
@@ -74,7 +74,8 @@ public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
@@ -84,7 +85,8 @@ public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT {
.setSchemaReplicationFactor(3)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java
index 97213150069..2114ed37c7f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java
@@ -66,14 +66,17 @@ public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualAutoIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
+
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setPipeAirGapReceiverEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
index 7fbb0f3a717..9a8ec2fc18e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
@@ -459,7 +459,6 @@ public class IoTDBPipeDataSinkIT extends AbstractPipeDualAutoIT {
testLoadTsFileWithoutVerify("async");
}
- @Test
private void testLoadTsFileWithoutVerify(final String loadTsFileStrategy) throws Exception {
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
index 0b4636c0d13..4b33de0a31c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
@@ -71,13 +71,15 @@ public class IoTDBPipeExtractorIT extends AbstractPipeDualAutoIT {
// Disable sender compaction for tsfile determination in loose range test
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
- .setEnableCrossSpaceCompaction(false);
+ .setEnableCrossSpaceCompaction(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java
index addf0314b40..771d50c97c6 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java
@@ -65,13 +65,15 @@ public class IoTDBPipeIdempotentIT extends AbstractPipeDualAutoIT {
// of the tested idempotent sql.
.setDefaultSchemaRegionGroupNumPerDatabase(1)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java
index 125fd6972b1..13a63a585a8 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java
@@ -59,13 +59,15 @@ public class IoTDBPipeProcessorIT extends AbstractPipeDualAutoIT {
.setAutoCreateSchemaEnabled(true)
.setTimestampPrecision("ms")
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java
index c5d41f001cb..1f646768525 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java
@@ -74,7 +74,8 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualAutoIT {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
- .setDataReplicationFactor(dataRegionReplicationFactor);
+ .setDataReplicationFactor(dataRegionReplicationFactor)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
@@ -83,7 +84,8 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualAutoIT {
.setSchemaRegionConsensusProtocolClass(schemaRegionConsensus)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
- .setDataReplicationFactor(dataRegionReplicationFactor);
+ .setDataReplicationFactor(dataRegionReplicationFactor)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java
index 563377c927e..c71e212b629 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java
@@ -61,13 +61,15 @@ public class IoTDBPipeWithLoadIT extends AbstractPipeDualAutoIT {
// Disable sender compaction to test mods
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
- .setEnableCrossSpaceCompaction(false);
+ .setEnableCrossSpaceCompaction(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java
index 3e9f5111165..b7091a1db32 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java
@@ -48,13 +48,15 @@ abstract class AbstractPipeDualManualIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
index 4c513576488..9a484426d76 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
@@ -36,6 +36,7 @@ abstract class AbstractPipeSingleIT {
env.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
// 10 min, assert that the operations will not time out
env.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+ env.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
env.initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
index de17dec1ed4..8156139b84c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
@@ -82,7 +82,8 @@ public class IoTDBSubscriptionRestartIT extends AbstractSubscriptionIT {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(3)
- .setDataReplicationFactor(2);
+ .setDataReplicationFactor(2)
+ .setIsPipeEnableMemoryCheck(false);
EnvFactory.getEnv().initClusterEnvironment(3, 3);
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
index 594f9efe691..5e61607ff32 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
@@ -58,6 +58,9 @@ public abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java
index 6f8cba43e79..5ad46b10977 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionConsumerGroupIT.java
@@ -117,8 +117,11 @@ public class IoTDBSubscriptionConsumerGroupIT extends AbstractSubscriptionDualIT
protected void setUpConfig() {
super.setUpConfig();
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+
// Enable air gap receiver
receiverEnv.getConfig().getCommonConfig().setPipeAirGapReceiverEnabled(true);
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java
index 4327b7a00f0..1939a986c68 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTimePrecisionIT.java
@@ -68,7 +68,9 @@ public class IoTDBSubscriptionTimePrecisionIT extends AbstractSubscriptionDualIT
// Set timestamp precision to nanosecond
senderEnv.getConfig().getCommonConfig().setTimestampPrecision("ns");
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
receiverEnv.getConfig().getCommonConfig().setTimestampPrecision("ns");
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Test
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java
index b046f09a6da..37971833b97 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/IoTDBSubscriptionTopicIT.java
@@ -91,6 +91,9 @@ public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT {
.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30);
senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerInitialSyncDelayMinutes(1);
senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerSyncIntervalMinutes(1);
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Test
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
index 3f415051928..7b564023498 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
@@ -33,7 +33,12 @@ public abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT
super.setUp();
// enable subscription
- EnvFactory.getEnv().getConfig().getCommonConfig().setSubscriptionEnabled(true);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSubscriptionEnabled(true)
+ .setIsPipeEnableMemoryCheck(false);
+
EnvFactory.getEnv().initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
index 3b49eb8ed13..7c2a969835b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
@@ -69,6 +69,10 @@ public abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionI
sender.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiver1.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiver2.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+
+ sender.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiver1.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiver2.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
index 6ad0c843e27..115dafbc5dc 100644
--- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
@@ -53,7 +53,11 @@ public class ExportTsFileTestIT extends AbstractScriptIT {
@BeforeClass
public static void setUp() throws Exception {
// enable subscription
- EnvFactory.getEnv().getConfig().getCommonConfig().setSubscriptionEnabled(true);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSubscriptionEnabled(true)
+ .setIsPipeEnableMemoryCheck(false);
EnvFactory.getEnv().initClusterEnvironment();
ip = EnvFactory.getEnv().getIP();
diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
index 3d041867242..4c7fffcfba5 100644
--- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
+++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.pipe.api.event.dml.insertion;
import org.apache.iotdb.pipe.api.event.Event;
+import java.io.File;
+
/**
* {@link TsFileInsertionEvent} is used to define the event of writing TsFile. Event data stores in
* disks, which is compressed and encoded, and requires IO cost for computational processing.
@@ -34,4 +36,12 @@ public interface TsFileInsertionEvent extends Event, AutoCloseable {
* @return {@code Iterable<TabletInsertionEvent>} the list of {@link TabletInsertionEvent}
*/
Iterable<TabletInsertionEvent> toTabletInsertionEvents();
+
+ /**
+ * Get the file that stores the data of this {@link TsFileInsertionEvent}. The file is compressed
+ * and encoded, and requires IO cost for computational processing.
+ *
+ * @return the file that stores the data of this {@link TsFileInsertionEvent}
+ */
+ File getTsFile();
}
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 81f136fadd0..34f9872bb10 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -259,6 +259,7 @@ public enum TSStatusCode {
PIPE_RECEIVER_USER_CONFLICT_EXCEPTION(1810),
PIPE_CONFIG_RECEIVER_HANDSHAKE_NEEDED(1811),
PIPE_TRANSFER_SLICE_OUT_OF_ORDER(1812),
+ PIPE_PUSH_META_TIMEOUT(1813),
// Subscription
SUBSCRIPTION_VERSION_ERROR(1900),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
index 0e90174a7f2..bf145085c13 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
@@ -46,11 +46,13 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
@@ -250,4 +252,10 @@ public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
resp.setPipeRemainingEventCountList(pipeRemainingEventCountList);
resp.setPipeRemainingTimeList(pipeRemainingTimeList);
}
+
+ @Override
+ public void runPipeTasks(
+ final Collection<PipeTask> pipeTasks, final Consumer<PipeTask> runSingle) {
+ pipeTasks.forEach(runSingle);
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
index 3ab75270b0b..eb4ffc48c79 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
@@ -422,12 +422,20 @@ public abstract class AbstractOperatePipeProcedureV2
* @return Error messages for the given pipe after pushing pipe meta
*/
public static String parsePushPipeMetaExceptionForPipe(
- String pipeName, Map<Integer, TPushPipeMetaResp> respMap) {
+ final String pipeName, final Map<Integer, TPushPipeMetaResp> respMap) {
final StringBuilder exceptionMessageBuilder = new StringBuilder();
- for (Map.Entry<Integer, TPushPipeMetaResp> respEntry : respMap.entrySet()) {
- int dataNodeId = respEntry.getKey();
- TPushPipeMetaResp resp = respEntry.getValue();
+ for (final Map.Entry<Integer, TPushPipeMetaResp> respEntry : respMap.entrySet()) {
+ final int dataNodeId = respEntry.getKey();
+ final TPushPipeMetaResp resp = respEntry.getValue();
+
+ if (resp.getStatus().getCode() == TSStatusCode.PIPE_PUSH_META_TIMEOUT.getStatusCode()) {
+ exceptionMessageBuilder.append(
+ String.format(
+ "DataNodeId: %s, Message: Timeout to wait for lock while processing pushPipeMeta on dataNodes.",
+ dataNodeId));
+ continue;
+ }
if (resp.getStatus().getCode() == TSStatusCode.PIPE_PUSH_META_ERROR.getStatusCode()) {
if (!resp.isSetExceptionMessages()) {
@@ -438,7 +446,7 @@ public abstract class AbstractOperatePipeProcedureV2
continue;
}
- AtomicBoolean hasException = new AtomicBoolean(false);
+ final AtomicBoolean hasException = new AtomicBoolean(false);
resp.getExceptionMessages()
.forEach(
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
index c2c04f94529..8854ee647f9 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
@@ -44,7 +44,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getTimestampPrecision(), deserializeReq.getTimestampPrecision());
}
@@ -57,7 +56,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
}
@Test
@@ -72,7 +70,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileName(), deserializeReq.getFileName());
Assert.assertEquals(req.getStartWritingOffset(), deserializeReq.getStartWritingOffset());
@@ -95,7 +92,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileNames(), deserializeReq.getFileNames());
Assert.assertEquals(req.getFileLengths(), deserializeReq.getFileLengths());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 446a428eee7..890af218a59 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -314,6 +314,8 @@ public class IoTDBConfig {
private String extPipeDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_PIPE_FOLDER_NAME;
+ private int pipeTaskThreadCount = 5;
+
/** External lib directory for MQTT, stores user-uploaded JAR files */
private String mqttDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.MQTT_FOLDER_NAME;
@@ -1046,6 +1048,8 @@ public class IoTDBConfig {
/** Policy of DataNodeSchemaCache eviction */
private String dataNodeSchemaCacheEvictionPolicy = "FIFO";
+ private int schemaThreadCount = 5;
+
private String readConsistencyLevel = "strong";
/** Maximum execution time of a DriverTask */
@@ -1147,8 +1151,6 @@ public class IoTDBConfig {
private long loadTsFileTabletConversionBatchMemorySizeInBytes = 4096 * 1024;
- private int loadTsFileTabletConversionThreadCount = 5;
-
private long loadChunkMetadataMemorySizeInBytes = 33554432; // 32MB
private long loadMemoryAllocateRetryIntervalMs = 1000L;
@@ -3345,6 +3347,14 @@ public class IoTDBConfig {
this.extPipeDir = extPipeDir;
}
+ public int getPipeTaskThreadCount() {
+ return pipeTaskThreadCount;
+ }
+
+ public void setPipeTaskThreadCount(int pipeTaskThreadCount) {
+ this.pipeTaskThreadCount = pipeTaskThreadCount;
+ }
+
public void setPartitionCacheSize(int partitionCacheSize) {
this.partitionCacheSize = partitionCacheSize;
}
@@ -3481,6 +3491,14 @@ public class IoTDBConfig {
this.dataNodeSchemaCacheEvictionPolicy = dataNodeSchemaCacheEvictionPolicy;
}
+ public int getSchemaThreadCount() {
+ return schemaThreadCount;
+ }
+
+ public void setSchemaThreadCount(int schemaThreadCount) {
+ this.schemaThreadCount = schemaThreadCount;
+ }
+
public String getReadConsistencyLevel() {
return readConsistencyLevel;
}
@@ -4026,14 +4044,6 @@ public class IoTDBConfig {
loadTsFileTabletConversionBatchMemorySizeInBytes;
}
- public int getLoadTsFileTabletConversionThreadCount() {
- return loadTsFileTabletConversionThreadCount;
- }
-
- public void setLoadTsFileTabletConversionThreadCount(int loadTsFileTabletConversionThreadCount) {
- this.loadTsFileTabletConversionThreadCount = loadTsFileTabletConversionThreadCount;
- }
-
public long getLoadChunkMetadataMemorySizeInBytes() {
return loadChunkMetadataMemorySizeInBytes;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 25f3095d9a0..7fbd9ce2ed7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -955,6 +955,10 @@ public class IoTDBDescriptor {
}
conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir()).trim());
+ conf.setPipeTaskThreadCount(
+ Integer.parseInt(
+ properties.getProperty(
+ "pipe_task_thread_count", Integer.toString(conf.getPipeTaskThreadCount()).trim())));
// At the same time, set TSFileConfig
List<FSType> fsTypes = new ArrayList<>();
@@ -1091,6 +1095,11 @@ public class IoTDBDescriptor {
properties.getProperty(
"datanode_schema_cache_eviction_policy", conf.getDataNodeSchemaCacheEvictionPolicy()));
+ conf.setSchemaThreadCount(
+ Integer.parseInt(
+ properties.getProperty(
+ "schema_thread_count", Integer.toString(conf.getSchemaThreadCount()))));
+
loadIoTConsensusProps(properties);
loadPipeConsensusProps(properties);
@@ -2386,11 +2395,6 @@ public class IoTDBDescriptor {
properties.getProperty(
"load_tsfile_tablet_conversion_batch_memory_size_in_bytes",
String.valueOf(conf.getLoadTsFileTabletConversionBatchMemorySizeInBytes()))));
- conf.setLoadTsFileTabletConversionThreadCount(
- Integer.parseInt(
- properties.getProperty(
- "load_tsfile_tablet_conversion_thread_count",
- String.valueOf(conf.getLoadTsFileTabletConversionThreadCount()))));
conf.setLoadChunkMetadataMemorySizeInBytes(
Long.parseLong(
Optional.ofNullable(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
index 96e2e4e7299..089d4e2fda7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
@@ -36,7 +36,6 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
import org.apache.iotdb.db.service.ResourcesInformationHolder;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
import org.apache.iotdb.pipe.api.exception.PipeException;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -161,7 +160,6 @@ class PipeAgentLauncher {
try (final ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
final TGetAllPipeInfoResp getAllPipeInfoResp = configNodeClient.getAllPipeInfo();
- WALInsertNodeCache.init();
PipeTabletEventBatch.init();
if (getAllPipeInfoResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
throw new StartupException("Failed to get pipe task meta from config node.");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
index 5a1f9d8b845..afb05fc2c39 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
@@ -35,7 +35,6 @@ import org.apache.iotdb.commons.service.ServiceType;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningQueue;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
@@ -86,15 +85,6 @@ public class PipeDataNodeRuntimeAgent implements IService {
PipeConfig.getInstance().printAllConfigs();
PipeAgentLauncher.launchPipeTaskAgent();
- registerPeriodicalJob(
- "PipeTaskAgent#restartAllStuckPipes",
- PipeDataNodeAgent.task()::restartAllStuckPipes,
- PipeConfig.getInstance().getPipeStuckRestartIntervalSeconds());
- registerPeriodicalJob(
- "PipeTaskAgent#flushDataRegionIfNeeded",
- PipeTerminateEvent::flushDataRegionIfNeeded,
- PipeConfig.getInstance().getPipeFlushAfterLastTerminateSeconds());
-
pipePeriodicalJobExecutor.start();
if (PipeConfig.getInstance().getPipeEventReferenceTrackingEnabled()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
index ef0d4c6b4d9..83854f2e4ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.db.pipe.agent.task;
+import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.consensus.SchemaRegionId;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
@@ -34,8 +38,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeType;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant;
import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
-import org.apache.iotdb.commons.service.metric.MetricService;
-import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -45,22 +48,17 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.agent.task.builder.PipeDataNodeBuilder;
import org.apache.iotdb.db.pipe.agent.task.builder.PipeDataNodeTaskBuilder;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
-import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningFilter;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
-import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.SystemMetric;
import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp;
import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq;
import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatResp;
@@ -78,6 +76,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -86,21 +85,52 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_START_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATH_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATTERN_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_START_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_START_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_PATH_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_PATTERN_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_START_TIME_KEY;
+
public class PipeDataNodeTaskAgent extends PipeTaskAgent {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeDataNodeTaskAgent.class);
protected static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final AtomicLong LAST_FORCED_RESTART_TIME =
- new AtomicLong(System.currentTimeMillis());
- private static final Map<String, AtomicLong> PIPE_NAME_TO_LAST_RESTART_TIME_MAP =
- new ConcurrentHashMap<>();
+ private final ExecutorService pipeExecutor =
+ new WrappedThreadPoolExecutor(
+ 0,
+ IoTDBDescriptor.getInstance().getConfig().getPipeTaskThreadCount(),
+ 0L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount()),
+ new IoTThreadFactory(ThreadName.PIPE_PARALLEL_EXECUTION_POOL.getName()),
+ ThreadName.PIPE_PARALLEL_EXECUTION_POOL.getName(),
+ new ThreadPoolExecutor.CallerRunsPolicy());
////////////////////////// Pipe Task Management Entry //////////////////////////
@@ -115,54 +145,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
return new PipeDataNodeBuilder(pipeMetaFromConfigNode).build();
}
- ////////////////////////// Manage by Pipe Name //////////////////////////
-
- @Override
- protected void startPipe(final String pipeName, final long creationTime) {
- final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName);
- final PipeStatus status = existedPipeMeta.getRuntimeMeta().getStatus().get();
- if (PipeStatus.STOPPED.equals(status) || status == null) {
- restartPipeToReloadResourceIfNeeded(existedPipeMeta);
- }
-
- super.startPipe(pipeName, creationTime);
- }
-
- private void restartPipeToReloadResourceIfNeeded(final PipeMeta pipeMeta) {
- if (System.currentTimeMillis() - pipeMeta.getStaticMeta().getCreationTime()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()) {
- return;
- }
-
- final AtomicLong lastRestartTime =
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP.get(pipeMeta.getStaticMeta().getPipeName());
- if (lastRestartTime != null
- && System.currentTimeMillis() - lastRestartTime.get()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()) {
- LOGGER.info(
- "Skipping reload resource for stopped pipe {} before starting it because reloading resource is too frequent.",
- pipeMeta.getStaticMeta().getPipeName());
- return;
- }
-
- if (PIPE_NAME_TO_LAST_RESTART_TIME_MAP.isEmpty()) {
- LOGGER.info(
- "Flushing storage engine before restarting pipe {}.",
- pipeMeta.getStaticMeta().getPipeName());
- final long currentTime = System.currentTimeMillis();
- StorageEngine.getInstance().syncCloseAllProcessor();
- WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes();
- LOGGER.info(
- "Finished flushing storage engine, time cost: {} ms.",
- System.currentTimeMillis() - currentTime);
- }
-
- restartStuckPipe(pipeMeta);
- LOGGER.info(
- "Reloaded resource for stopped pipe {} before starting it.",
- pipeMeta.getStaticMeta().getPipeName());
- }
-
///////////////////////// Manage by regionGroupId /////////////////////////
@Override
@@ -306,13 +288,12 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
@Override
protected void thawRate(final String pipeName, final long creationTime) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance().thawRate(pipeName + "_" + creationTime);
+ PipeDataNodeSinglePipeMetrics.getInstance().thawRate(pipeName + "_" + creationTime);
}
@Override
protected void freezeRate(final String pipeName, final long creationTime) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
- .freezeRate(pipeName + "_" + creationTime);
+ PipeDataNodeSinglePipeMetrics.getInstance().freezeRate(pipeName + "_" + creationTime);
}
@Override
@@ -323,7 +304,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
final String taskId = pipeName + "_" + creationTime;
PipeTsFileToTabletsMetrics.getInstance().deregister(taskId);
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance().deregister(taskId);
+ PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId);
return true;
}
@@ -351,7 +332,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
final long creationTime = pipeMeta.getStaticMeta().getCreationTime();
final String taskId = pipeName + "_" + creationTime;
PipeTsFileToTabletsMetrics.getInstance().deregister(taskId);
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance().deregister(taskId);
+ PipeDataNodeSinglePipeMetrics.getInstance().deregister(taskId);
// When the pipe contains no pipe tasks, there is no corresponding prefetching queue for the
// subscribed pipe, so the subscription needs to be manually marked as completed.
if (!hasPipeTasks && PipeStaticMeta.isSubscriptionPipe(pipeName)) {
@@ -379,9 +360,8 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
///////////////////////// Heartbeat /////////////////////////
public void collectPipeMetaList(final TDataNodeHeartbeatResp resp) throws TException {
- // Try the lock instead of directly acquire it to prevent the block of the cluster heartbeat
- // 10s is the half of the HEARTBEAT_TIMEOUT_TIME defined in class BaseNodeCache in ConfigNode
- if (!tryReadLockWithTimeOut(10)) {
+ if (!tryReadLockWithTimeOut(
+ CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS() * 2L / 3)) {
return;
}
try {
@@ -445,7 +425,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
final boolean isCompleted = isAllDataRegionCompleted && includeDataAndNeedDrop;
final Pair<Long, Double> remainingEventAndTime =
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.getRemainingEventAndTime(staticMeta.getPipeName(), staticMeta.getCreationTime());
pipeCompletedList.add(isCompleted);
pipeRemainingEventCountList.add(remainingEventAndTime.getLeft());
@@ -528,7 +508,7 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
final boolean isCompleted = isAllDataRegionCompleted && includeDataAndNeedDrop;
final Pair<Long, Double> remainingEventAndTime =
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.getRemainingEventAndTime(staticMeta.getPipeName(), staticMeta.getCreationTime());
pipeCompletedList.add(isCompleted);
pipeRemainingEventCountList.add(remainingEventAndTime.getLeft());
@@ -554,226 +534,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
PipeInsertionDataNodeListener.getInstance().listenToHeartbeat(true);
}
- ///////////////////////// Restart Logic /////////////////////////
-
- public void restartAllStuckPipes() {
- final List<String> removedPipeName = removeOutdatedPipeInfoFromLastRestartTimeMap();
- if (!removedPipeName.isEmpty()) {
- final long currentTime = System.currentTimeMillis();
- LOGGER.info(
- "Pipes {} now can dynamically adjust their extraction strategies. "
- + "Start to flush storage engine to trigger the adjustment.",
- removedPipeName);
- StorageEngine.getInstance().syncCloseAllProcessor();
- WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes();
- LOGGER.info(
- "Finished flushing storage engine, time cost: {} ms.",
- System.currentTimeMillis() - currentTime);
- LOGGER.info("Skipping restarting pipes this round because of the dynamic flushing.");
- return;
- }
-
- if (!tryWriteLockWithTimeOut(5)) {
- return;
- }
-
- final Set<PipeMeta> stuckPipes;
- try {
- stuckPipes = findAllStuckPipes();
- } finally {
- releaseWriteLock();
- }
-
- // If the pipe has been restarted recently, skip it.
- stuckPipes.removeIf(
- pipeMeta -> {
- final AtomicLong lastRestartTime =
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP.get(pipeMeta.getStaticMeta().getPipeName());
- return lastRestartTime != null
- && System.currentTimeMillis() - lastRestartTime.get()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs();
- });
-
- // Restart all stuck pipes.
- // Note that parallelStream cannot be used here. The method PipeTaskAgent#dropPipe also uses
- // parallelStream. If parallelStream is used here, the subtasks generated inside the dropPipe
- // may not be scheduled by the worker thread of ForkJoinPool because of less available threads,
- // and the parent task will wait for the completion of the subtasks in ForkJoinPool forever,
- // causing the deadlock.
- stuckPipes.forEach(this::restartStuckPipe);
- }
-
- private List<String> removeOutdatedPipeInfoFromLastRestartTimeMap() {
- final List<String> removedPipeName = new ArrayList<>();
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP
- .entrySet()
- .removeIf(
- entry -> {
- final AtomicLong lastRestartTime = entry.getValue();
- final boolean shouldRemove =
- lastRestartTime == null
- || PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()
- <= System.currentTimeMillis() - lastRestartTime.get();
- if (shouldRemove) {
- removedPipeName.add(entry.getKey());
- }
- return shouldRemove;
- });
- return removedPipeName;
- }
-
- private Set<PipeMeta> findAllStuckPipes() {
- final Set<PipeMeta> stuckPipes = new HashSet<>();
-
- if (System.currentTimeMillis() - LAST_FORCED_RESTART_TIME.get()
- > PipeConfig.getInstance().getPipeSubtaskExecutorForcedRestartIntervalMs()) {
- LAST_FORCED_RESTART_TIME.set(System.currentTimeMillis());
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- stuckPipes.add(pipeMeta);
- }
- if (!stuckPipes.isEmpty()) {
- LOGGER.warn(
- "All {} pipe(s) will be restarted because of forced restart policy.",
- stuckPipes.size());
- }
- return stuckPipes;
- }
-
- final long totalLinkedButDeletedTsFileResourceRamSize =
- PipeDataNodeResourceManager.tsfile().getTotalLinkedButDeletedTsFileResourceRamSize();
- final long totalInsertNodeFloatingMemoryUsageInBytes = getAllFloatingMemoryUsageInByte();
- final long totalFloatingMemorySizeInBytes =
- PipeMemoryManager.getTotalFloatingMemorySizeInBytes();
- if (totalInsertNodeFloatingMemoryUsageInBytes + totalLinkedButDeletedTsFileResourceRamSize
- >= totalFloatingMemorySizeInBytes) {
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- stuckPipes.add(pipeMeta);
- }
- if (!stuckPipes.isEmpty()) {
- LOGGER.warn(
- "All {} pipe(s) will be restarted because linked but deleted tsFiles' resource size {} and all insertNode's size {} exceeds limit {}.",
- stuckPipes.size(),
- totalLinkedButDeletedTsFileResourceRamSize,
- totalInsertNodeFloatingMemoryUsageInBytes,
- totalFloatingMemorySizeInBytes);
- }
- return stuckPipes;
- }
-
- final Map<String, IoTDBDataRegionExtractor> taskId2ExtractorMap =
- PipeDataRegionExtractorMetrics.getInstance().getExtractorMap();
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- final String pipeName = pipeMeta.getStaticMeta().getPipeName();
- final List<IoTDBDataRegionExtractor> extractors =
- taskId2ExtractorMap.values().stream()
- .filter(e -> e.getPipeName().equals(pipeName) && e.shouldExtractInsertion())
- .collect(Collectors.toList());
-
- if (extractors.isEmpty()) {
- continue;
- }
-
- // Extractors of this pipe might not pin too much MemTables,
- // still need to check if linked-and-deleted TsFile count exceeds limit.
- // Typically, if deleted tsFiles are too abundant all pipes may need to restart.
- if ((CONFIG.isEnableSeqSpaceCompaction()
- || CONFIG.isEnableUnseqSpaceCompaction()
- || CONFIG.isEnableCrossSpaceCompaction())
- && mayDeletedTsFileSizeReachDangerousThreshold()) {
- LOGGER.warn(
- "Pipe {} needs to restart because too many TsFiles are out-of-date.",
- pipeMeta.getStaticMeta());
- stuckPipes.add(pipeMeta);
- continue;
- }
-
- // Try to restart the stream mode pipes for releasing memTables.
- if (extractors.get(0).isStreamMode()) {
- if (extractors.stream().anyMatch(IoTDBDataRegionExtractor::hasConsumedAllHistoricalTsFiles)
- && (mayMemTablePinnedCountReachDangerousThreshold()
- || mayWalSizeReachThrottleThreshold())) {
- // Extractors of this pipe may be stuck and is pinning too many MemTables.
- LOGGER.warn(
- "Pipe {} needs to restart because too many memTables are pinned or the WAL size is too large. mayMemTablePinnedCountReachDangerousThreshold: {}, mayWalSizeReachThrottleThreshold: {}",
- pipeMeta.getStaticMeta(),
- mayMemTablePinnedCountReachDangerousThreshold(),
- mayWalSizeReachThrottleThreshold());
- stuckPipes.add(pipeMeta);
- }
- }
- }
-
- return stuckPipes;
- }
-
- private boolean mayDeletedTsFileSizeReachDangerousThreshold() {
- try {
- final long linkedButDeletedTsFileSize =
- PipeDataNodeResourceManager.tsfile().getTotalLinkedButDeletedTsfileSize();
- final double totalDisk =
- MetricService.getInstance()
- .getAutoGauge(
- SystemMetric.SYS_DISK_TOTAL_SPACE.toString(),
- MetricLevel.CORE,
- Tag.NAME.toString(),
- // This "system" should stay the same with the one in
- // DataNodeInternalRPCServiceImpl.
- "system")
- .getValue();
- return linkedButDeletedTsFileSize > 0
- && totalDisk > 0
- && linkedButDeletedTsFileSize
- > PipeConfig.getInstance().getPipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage()
- * totalDisk;
- } catch (final Exception e) {
- LOGGER.warn("Failed to judge if deleted TsFile size reaches dangerous threshold.", e);
- return false;
- }
- }
-
- private boolean mayMemTablePinnedCountReachDangerousThreshold() {
- return PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount() != Integer.MAX_VALUE
- && PipeDataNodeResourceManager.wal().getPinnedWalCount()
- >= 5
- * PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber();
- }
-
- private boolean mayWalSizeReachThrottleThreshold() {
- return 3 * WALManager.getInstance().getTotalDiskUsage() > 2 * CONFIG.getThrottleThreshold();
- }
-
- private void restartStuckPipe(final PipeMeta pipeMeta) {
- LOGGER.warn(
- "Pipe {} will be restarted because it is stuck or has encountered issues such as data backlog or being stopped for too long.",
- pipeMeta.getStaticMeta());
- acquireWriteLock();
- try {
- final long startTime = System.currentTimeMillis();
- final PipeMeta originalPipeMeta = pipeMeta.deepCopy4TaskAgent();
- handleDropPipe(pipeMeta.getStaticMeta().getPipeName());
-
- final long restartTime = System.currentTimeMillis();
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP
- .computeIfAbsent(pipeMeta.getStaticMeta().getPipeName(), k -> new AtomicLong(restartTime))
- .set(restartTime);
- handleSinglePipeMetaChanges(originalPipeMeta);
-
- LOGGER.warn(
- "Pipe {} was restarted because of stuck or data backlog, time cost: {} ms.",
- originalPipeMeta.getStaticMeta(),
- System.currentTimeMillis() - startTime);
- } catch (final Exception e) {
- LOGGER.warn("Failed to restart stuck pipe {}.", pipeMeta.getStaticMeta(), e);
- } finally {
- releaseWriteLock();
- }
- }
-
- public boolean isPipeTaskCurrentlyRestarted(final String pipeName) {
- return PIPE_NAME_TO_LAST_RESTART_TIME_MAP.containsKey(pipeName);
- }
-
///////////////////////// Terminate Logic /////////////////////////
public void markCompleted(final String pipeName, final int regionId) {
@@ -817,6 +577,48 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
}
}
+ @Override
+ public void runPipeTasks(
+ final Collection<PipeTask> pipeTasks, final Consumer<PipeTask> runSingle) {
+ final Set<Future<?>> pipeFuture = new HashSet<>();
+
+ pipeTasks.forEach(
+ pipeTask -> pipeFuture.add(pipeExecutor.submit(() -> runSingle.accept(pipeTask))));
+
+ for (final Future<?> future : pipeFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing pipe task: ", e);
+ throw new PipeException(e.toString());
+ }
+ }
+ }
+
+ ///////////////////////// Shutdown Logic /////////////////////////
+
+ public void persistAllProgressIndexLocally() {
+ if (!PipeConfig.getInstance().isPipeProgressIndexPersistEnabled()) {
+ LOGGER.info(
+ "Pipe progress index persist disabled. Skipping persist all progress index locally.");
+ return;
+ }
+ if (!tryReadLockWithTimeOut(10)) {
+ LOGGER.info("Failed to persist all progress index locally because of timeout.");
+ return;
+ }
+ try {
+ for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
+ pipeMeta.getRuntimeMeta().persistProgressIndex();
+ }
+ LOGGER.info("Persist all progress index locally successfully.");
+ } catch (final Exception e) {
+ LOGGER.warn("Failed to record all progress index locally, because {}.", e.getMessage(), e);
+ } finally {
+ releaseReadLock();
+ }
+ }
+
///////////////////////// Pipe Consensus /////////////////////////
public ProgressIndex getPipeTaskProgressIndex(final String pipeName, final int consensusGroupId) {
@@ -859,4 +661,203 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
releaseReadLock();
}
}
+
+ @Override
+ protected void calculateMemoryUsage(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+ if (!PipeConfig.getInstance().isPipeEnableMemoryCheck()) {
+ return;
+ }
+
+ calculateInsertNodeQueueMemory(extractorParameters, processorParameters, connectorParameters);
+
+ long needMemory = 0;
+
+ needMemory +=
+ calculateTsFileParserMemory(extractorParameters, processorParameters, connectorParameters);
+ needMemory +=
+ calculateSinkBatchMemory(extractorParameters, processorParameters, connectorParameters);
+ needMemory +=
+ calculateSendTsFileReadBufferMemory(
+ extractorParameters, processorParameters, connectorParameters);
+
+ PipeMemoryManager pipeMemoryManager = PipeDataNodeResourceManager.memory();
+ final long freeMemorySizeInBytes = pipeMemoryManager.getFreeMemorySizeInBytes();
+ final long reservedMemorySizeInBytes =
+ (long)
+ (PipeMemoryManager.getTotalMemorySizeInBytes()
+ * PipeConfig.getInstance().getReservedMemoryPercentage());
+ if (freeMemorySizeInBytes < needMemory + reservedMemorySizeInBytes) {
+ final String message =
+ String.format(
+ "Not enough memory for pipe. Need memory: %d bytes, free memory: %d bytes, reserved memory: %d bytes, total memory: %d bytes",
+ needMemory,
+ freeMemorySizeInBytes,
+ freeMemorySizeInBytes,
+ PipeMemoryManager.getTotalMemorySizeInBytes());
+ LOGGER.warn(message);
+ throw new PipeException(message);
+ }
+ }
+
+ private void calculateInsertNodeQueueMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // Realtime extractor is enabled by default, so we only need to check the source realtime
+ if (!extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
+ EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)) {
+ return;
+ }
+
+ // If the realtime mode is batch or file, we do not need to allocate memory
+ final String realtimeMode =
+ extractorParameters.getStringByKeys(
+ PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_KEY,
+ PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY);
+ if (PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE.equals(realtimeMode)
+ || PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FILE_VALUE.equals(realtimeMode)) {
+ return;
+ }
+
+ final long allocatedMemorySizeInBytes = this.getAllFloatingMemoryUsageInByte();
+ final long remainingMemory =
+ PipeMemoryManager.getTotalFloatingMemorySizeInBytes() - allocatedMemorySizeInBytes;
+ if (remainingMemory < PipeConfig.getInstance().PipeInsertNodeQueueMemory()) {
+ final String message =
+ String.format(
+ "Not enough memory for pipe. Need Floating memory: %d bytes, free Floating memory: %d bytes",
+ PipeConfig.getInstance().PipeInsertNodeQueueMemory(), remainingMemory);
+ LOGGER.warn(message);
+ throw new PipeException(message);
+ }
+ }
+
+ private long calculateTsFileParserMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // If the extractor is not history, we do not need to allocate memory
+ boolean isExtractorHistory =
+ extractorParameters.getBooleanOrDefault(
+ SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE);
+
+ // If the extractor is history, and has start/end time, we need to allocate memory
+ boolean isTSFileParser =
+ isExtractorHistory
+ && extractorParameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY);
+
+ isTSFileParser =
+ isTSFileParser
+ || (isExtractorHistory
+ && extractorParameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY));
+
+ // if the extractor has start/end time, we need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(
+ SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY);
+
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY);
+
+ // If the extractor has pattern or path, we need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY);
+
+ isTSFileParser =
+ isTSFileParser || extractorParameters.hasAnyAttributes(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY);
+
+ // If the extractor is not hybrid, we do need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || !PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals(
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY,
+ PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE));
+
+ if (!isTSFileParser) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getTsFileParserMemory();
+ }
+
+ private long calculateSinkBatchMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // If the connector format is tsfile , we need to use batch
+ boolean needUseBatch =
+ PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE.equals(
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY,
+ PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE));
+
+ if (needUseBatch) {
+ return PipeConfig.getInstance().getSinkBatchMemoryTsFile();
+ }
+
+ // If the connector is batch mode, we need to use batch
+ needUseBatch =
+ connectorParameters.getBooleanOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY,
+ PipeConnectorConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY),
+ PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE);
+
+ if (!needUseBatch) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getSinkBatchMemoryInsertNode();
+ }
+
+ private long calculateSendTsFileReadBufferMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+ // If the extractor is history enable, we need to transfer tsfile
+ boolean needTransferTsFile =
+ extractorParameters.getBooleanOrDefault(
+ SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE);
+
+ String format =
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY, PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE);
+
+ // If the connector format is tsfile and hybrid, we need to transfer tsfile
+ needTransferTsFile =
+ needTransferTsFile
+ || PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals(format)
+ || PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE.equals(format);
+
+ if (!needTransferTsFile) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getSendTsFileReadBuffer();
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
index ecbbc641e4b..6e4a858a370 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.connector;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.pipe.api.event.Event;
import org.slf4j.Logger;
@@ -65,6 +66,8 @@ public class PipeConnectorSubtaskLifeCycle implements AutoCloseable {
if (registeredTaskCount == 0) {
executor.register(subtask);
runningTaskCount = 0;
+
+ PipeDataNodeResourceManager.compaction().registerPipeConnectorSubtaskLifeCycle(this);
}
registeredTaskCount++;
@@ -152,5 +155,7 @@ public class PipeConnectorSubtaskLifeCycle implements AutoCloseable {
@Override
public synchronized void close() {
executor.deregister(subtask.getTaskID());
+
+ PipeDataNodeResourceManager.compaction().deregisterPipeConnectorSubtaskLifeCycle(this);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
index a4f05447eae..5dc0c4b6ddc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
@@ -21,23 +21,40 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.connector;
import org.apache.iotdb.commons.pipe.agent.task.connection.BlockingPendingQueue;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
+import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeCompactedTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQueue<Event> {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(PipeRealtimePriorityBlockingQueue.class);
+
private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
private final BlockingDeque<TsFileInsertionEvent> tsfileInsertEventDeque =
@@ -55,7 +72,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
}
@Override
- public boolean directOffer(final Event event) {
+ public synchronized boolean directOffer(final Event event) {
checkBeforeOffer(event);
if (event instanceof TsFileInsertionEvent) {
@@ -73,18 +90,18 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
}
@Override
- public boolean waitedOffer(final Event event) {
+ public synchronized boolean waitedOffer(final Event event) {
return directOffer(event);
}
@Override
- public boolean put(final Event event) {
+ public synchronized boolean put(final Event event) {
directOffer(event);
return true;
}
@Override
- public Event directPoll() {
+ public synchronized Event directPoll() {
Event event = null;
final int pollHistoricalTsFileThreshold =
PIPE_CONFIG.getPipeRealTimeQueuePollHistoricalTsFileThreshold();
@@ -129,7 +146,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
* available.
*/
@Override
- public Event waitedPoll() {
+ public synchronized Event waitedPoll() {
Event event = null;
final int pollHistoricalTsFileThreshold =
PIPE_CONFIG.getPipeRealTimeQueuePollHistoricalTsFileThreshold();
@@ -176,7 +193,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
}
@Override
- public Event peek() {
+ public synchronized Event peek() {
final Event event = pendingQueue.peek();
if (Objects.nonNull(event)) {
return event;
@@ -184,20 +201,135 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
return tsfileInsertEventDeque.peek();
}
+ public synchronized void replace(
+ String dataRegionId, Set<TsFileResource> sourceFiles, List<TsFileResource> targetFiles) {
+ // TODO: return if pipe sink is for pipe consensus
+
+ final int regionId = Integer.parseInt(dataRegionId);
+ final Map<CommitterKey, Set<PipeTsFileInsertionEvent>> eventsToBeRemovedGroupByCommitterKey =
+ tsfileInsertEventDeque.stream()
+ .filter(
+ event ->
+ event instanceof PipeTsFileInsertionEvent
+ && ((PipeTsFileInsertionEvent) event).getRegionId() == regionId)
+ .map(event -> (PipeTsFileInsertionEvent) event)
+ .collect(
+ Collectors.groupingBy(
+ PipeTsFileInsertionEvent::getCommitterKey, Collectors.toSet()))
+ .entrySet()
+ .stream()
+ // Replace if all source files are present in the queue
+ .filter(entry -> entry.getValue().size() == sourceFiles.size())
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ if (eventsToBeRemovedGroupByCommitterKey.isEmpty()) {
+ return;
+ }
+
+ final Map<CommitterKey, Set<PipeTsFileInsertionEvent>> eventsToBeAddedGroupByCommitterKey =
+ new HashMap<>();
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
+ eventsToBeRemovedGroupByCommitterKey.entrySet()) {
+ final CommitterKey committerKey = entry.getKey();
+ final PipeTsFileInsertionEvent anyEvent = entry.getValue().stream().findFirst().orElse(null);
+ final Set<PipeTsFileInsertionEvent> newEvents = new HashSet<>();
+ for (int i = 0; i < targetFiles.size(); i++) {
+ newEvents.add(
+ new PipeCompactedTsFileInsertionEvent(
+ committerKey,
+ entry.getValue(),
+ anyEvent,
+ targetFiles.get(i),
+ i == targetFiles.size() - 1));
+ }
+ eventsToBeAddedGroupByCommitterKey.put(committerKey, newEvents);
+ }
+
+ // Handling new events
+ final Set<PipeTsFileInsertionEvent> successfullyReferenceIncreasedEvents = new HashSet<>();
+ final AtomicBoolean
+ allSuccess = // To track if all events successfully increased the reference count
+ new AtomicBoolean(true);
+ outerLoop:
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> committerKeySetEntry :
+ eventsToBeAddedGroupByCommitterKey.entrySet()) {
+ for (final PipeTsFileInsertionEvent event : committerKeySetEntry.getValue()) {
+ if (event != null) {
+ try {
+ if (!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName())) {
+ allSuccess.set(false);
+ break outerLoop;
+ } else {
+ successfullyReferenceIncreasedEvents.add(event);
+ }
+ } catch (final Exception e) {
+ allSuccess.set(false);
+ break outerLoop;
+ }
+ }
+ }
+ }
+ if (!allSuccess.get()) {
+ // If any event failed to increase the reference count,
+ // we need to decrease the reference count for all successfully increased events
+ for (final PipeTsFileInsertionEvent event : successfullyReferenceIncreasedEvents) {
+ try {
+ event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to decrease reference count for event {} in PipeRealtimePriorityBlockingQueue",
+ event,
+ e);
+ }
+ }
+ return; // Exit early if any event failed to increase the reference count
+ } else {
+ // If all events successfully increased reference count,
+ // we can proceed to add them to the deque
+ for (final PipeTsFileInsertionEvent event : successfullyReferenceIncreasedEvents) {
+ tsfileInsertEventDeque.add(event);
+ eventCounter.increaseEventCount(event);
+ }
+ }
+
+ // Handling old events
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
+ eventsToBeRemovedGroupByCommitterKey.entrySet()) {
+ for (final PipeTsFileInsertionEvent event : entry.getValue()) {
+ if (event != null) {
+ try {
+ event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to decrease reference count for event {} in PipeRealtimePriorityBlockingQueue",
+ event,
+ e);
+ }
+ eventCounter.decreaseEventCount(event);
+ }
+ }
+ }
+ final Set<PipeTsFileInsertionEvent> eventsToRemove = new HashSet<>();
+ for (Set<PipeTsFileInsertionEvent> pipeTsFileInsertionEvents :
+ eventsToBeRemovedGroupByCommitterKey.values()) {
+ eventsToRemove.addAll(pipeTsFileInsertionEvents);
+ }
+ tsfileInsertEventDeque.removeIf(eventsToRemove::contains);
+ }
+
@Override
- public void clear() {
+ public synchronized void clear() {
super.clear();
tsfileInsertEventDeque.clear();
}
@Override
- public void forEach(final Consumer<? super Event> action) {
+ public synchronized void forEach(final Consumer<? super Event> action) {
super.forEach(action);
tsfileInsertEventDeque.forEach(action);
}
@Override
- public void discardAllEvents() {
+ public synchronized void discardAllEvents() {
super.discardAllEvents();
tsfileInsertEventDeque.removeIf(
event -> {
@@ -212,7 +344,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
}
@Override
- public void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) {
+ public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) {
super.discardEventsOfPipe(pipeNameToDrop, regionId);
tsfileInsertEventDeque.removeIf(
event -> {
@@ -244,7 +376,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
return tsfileInsertEventDeque.size();
}
- public void setOfferTsFileCounter(AtomicInteger offerTsFileCounter) {
+ public synchronized void setOfferTsFileCounter(AtomicInteger offerTsFileCounter) {
this.offerTsFileCounter = offerTsFileCounter;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java
index c8a0acf32fa..d39feb96fe0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java
@@ -31,7 +31,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.processor.PipeProcessorMetrics;
import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor;
import org.apache.iotdb.db.storageengine.StorageEngine;
@@ -140,7 +140,7 @@ public class PipeProcessorSubtask extends PipeReportableSubtask {
} else if (event instanceof TsFileInsertionEvent) {
pipeProcessor.process((TsFileInsertionEvent) event, outputEventCollector);
PipeProcessorMetrics.getInstance().markTsFileEvent(taskID);
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.markTsFileCollectInvocationCount(
pipeNameWithCreationTime, outputEventCollector.getCollectInvocationCount());
} else if (event instanceof PipeHeartbeatEvent) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
index 1cb9f50d92d..78a237a4102 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.pipe.connector.payload.evolvable.batch;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.memory.PipeDynamicMemoryBlock;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlockType;
import org.apache.iotdb.db.pipe.resource.memory.PipeModelFixedMemoryBlock;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
@@ -47,7 +48,7 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
private long firstEventProcessingTime = Long.MIN_VALUE;
protected long totalBufferSize = 0;
- private final PipeModelFixedMemoryBlock allocatedMemoryBlock;
+ private final PipeDynamicMemoryBlock allocatedMemoryBlock;
protected volatile boolean isClosed = false;
@@ -60,11 +61,9 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
// limit in buffer size
this.allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(
- requestMaxBatchSizeInBytes, PipeMemoryBlockType.BATCH);
+ pipeModelFixedMemoryBlock.registerPipeBatchMemoryBlock(requestMaxBatchSizeInBytes);
- if (getMaxBatchSizeInBytes() != requestMaxBatchSizeInBytes) {
+ if (getMaxBatchSizeInBytes() != allocatedMemoryBlock.getMemoryUsageInBytes()) {
LOGGER.info(
"PipeTabletEventBatch: the max batch size is adjusted from {} to {} due to the "
+ "memory restriction",
@@ -127,8 +126,12 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
throws WALPipeException, IOException;
public boolean shouldEmit() {
- return totalBufferSize >= getMaxBatchSizeInBytes()
- || System.currentTimeMillis() - firstEventProcessingTime >= maxDelayInMs;
+ final long diff = System.currentTimeMillis() - firstEventProcessingTime;
+ if (totalBufferSize >= getMaxBatchSizeInBytes() || diff >= maxDelayInMs) {
+ allocatedMemoryBlock.updateCurrentMemoryEfficiencyAdjustMem((double) diff / maxDelayInMs);
+ return true;
+ }
+ return false;
}
private long getMaxBatchSizeInBytes() {
@@ -194,9 +197,22 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
}
try {
- pipeModelFixedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(0L, PipeMemoryBlockType.BATCH);
+ long batchSize = PipeDataNodeResourceManager.memory().getAllocatedMemorySizeInBytesOfBatch();
+ for (long i = batchSize; i > 0; i = i / 2) {
+ try {
+ pipeModelFixedMemoryBlock =
+ PipeDataNodeResourceManager.memory()
+ .forceAllocateForModelFixedMemoryBlock(i, PipeMemoryBlockType.BATCH);
+
+ LOGGER.info("pipe model fixed memory block initialized with size: {} bytes", i);
+ return;
+ } catch (Exception ignore) {
+ // ignore the exception and try to allocate a smaller size
+ LOGGER.info(
+ "pipe model fixed memory block initialized with size: {} bytes failed, try smaller size",
+ i);
+ }
+ }
} catch (Exception e) {
LOGGER.error("init pipe model fixed memory block failed", e);
// If the allocation fails, we still need to create a default memory block to avoid NPE.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
index 292fc018ed7..33f228f42fb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
@@ -102,8 +102,7 @@ public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
(PipeInsertNodeTabletInsertionEvent) event;
// Read the bytebuffer from the wal file and transfer it directly without serializing or
// deserializing if possible
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
if (Objects.isNull(insertNode)) {
buffer = pipeInsertNodeTabletInsertionEvent.getByteBuffer();
binaryBuffers.add(buffer);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
index 580dbe2ed4c..553a04cbfcd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
@@ -66,7 +66,6 @@ public class PipeTransferPlanNodeReq extends TPipeTransferReq {
planNodeReq.version = transferReq.version;
planNodeReq.type = transferReq.type;
- planNodeReq.body = transferReq.body;
return planNodeReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
index 8090f650489..27664cdbac7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
@@ -182,7 +182,6 @@ public class PipeTransferTabletBatchReq extends TPipeTransferReq {
batchReq.version = transferReq.version;
batchReq.type = transferReq.type;
- batchReq.body = transferReq.body;
return batchReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
index 5e9e0a39103..4deac8a5cf6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
@@ -89,7 +89,6 @@ public class PipeTransferTabletBinaryReq extends TPipeTransferReq {
binaryReq.version = transferReq.version;
binaryReq.type = transferReq.type;
- binaryReq.body = transferReq.body;
return binaryReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
index c45417ba99d..376ca121c80 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
@@ -96,7 +96,6 @@ public class PipeTransferTabletInsertNodeReq extends TPipeTransferReq {
insertNodeReq.version = transferReq.version;
insertNodeReq.type = transferReq.type;
- insertNodeReq.body = transferReq.body;
return insertNodeReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
index 61790f883ae..f1e32785df7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
@@ -132,7 +132,6 @@ public class PipeTransferTabletRawReq extends TPipeTransferReq {
tabletReq.version = transferReq.version;
tabletReq.type = transferReq.type;
- tabletReq.body = transferReq.body;
return tabletReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
index ee35ab46306..bc4805e8c8e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
@@ -168,8 +168,7 @@ public class IoTDBDataRegionAirGapConnector extends IoTDBDataNodeAirGapConnector
final AirGapSocket socket,
final PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent)
throws PipeException, WALPipeException, IOException {
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final byte[] bytes =
Objects.isNull(insertNode)
? PipeTransferTabletBinaryReq.toTPipeTransferBytes(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
index a74057c75c7..1756da36db7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
@@ -277,8 +277,7 @@ public class PipeConsensusAsyncConnector extends IoTDBConnector implements Conse
return;
}
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final ProgressIndex progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
final TPipeConsensusTransferReq pipeConsensusTransferReq =
Objects.isNull(insertNode)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
index f4690783fe1..6186d9f671a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
@@ -252,7 +252,7 @@ public class PipeConsensusSyncConnector extends IoTDBConnector {
try (final SyncPipeConsensusServiceClient syncPipeConsensusServiceClient =
syncRetryClientManager.borrowClient(getFollowerUrl())) {
- insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
if (insertNode != null) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
index 870cea0ee87..a3f8fe557b7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
@@ -196,8 +196,7 @@ public abstract class PipeConsensusTransferBatchReqBuilder implements AutoClosea
// Read the bytebuffer from the wal file and transfer it directly without serializing or
// deserializing if possible
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
// PipeConsensus will transfer binary data to TPipeConsensusTransferReq
final ProgressIndex progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
if (Objects.isNull(insertNode)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
index fd80f47c45f..62871f039fb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
@@ -265,8 +265,7 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
return false;
}
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final TPipeTransferReq pipeTransferReq =
compressIfNeeded(
Objects.isNull(insertNode)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java
index f77d41b6662..b3eb223e58e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java
@@ -104,7 +104,7 @@ public abstract class PipeTransferTabletInsertionEventHandler extends PipeTransf
? ((EnrichedEvent) event).coreReportMessage()
: event.toString(),
event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitterKey() : null,
- event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitId() : null,
+ event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitIds() : null,
exception);
} finally {
connector.addFailureEventToRetryQueue(event);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
index ccce6ba0aa2..1c586ce04ec 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
@@ -285,7 +285,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
"Successfully transferred file {} (committer key={}, commit id={}, reference count={}).",
tsFile,
events.stream().map(EnrichedEvent::getCommitterKey).collect(Collectors.toList()),
- events.stream().map(EnrichedEvent::getCommitId).collect(Collectors.toList()),
+ events.stream().map(EnrichedEvent::getCommitIds).collect(Collectors.toList()),
referenceCount);
} else {
LOGGER.info(
@@ -350,7 +350,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
"Failed to transfer TsFileInsertionEvent {} (committer key {}, commit id {}).",
tsFile,
events.stream().map(EnrichedEvent::getCommitterKey).collect(Collectors.toList()),
- events.stream().map(EnrichedEvent::getCommitId).collect(Collectors.toList()),
+ events.stream().map(EnrichedEvent::getCommitIds).collect(Collectors.toList()),
exception);
} else {
LOGGER.warn(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
index 784ee14a55a..d996cac2820 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
@@ -294,8 +294,7 @@ public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
// getDeviceId() may return null for InsertRowsNode, will be equal to getClient(null)
clientAndStatus = clientManager.getClient(pipeInsertNodeTabletInsertionEvent.getDeviceId());
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final TPipeTransferReq req =
compressIfNeeded(
insertNode != null
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
index 7d28f8609ad..365950c5109 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
@@ -129,8 +129,7 @@ public class WriteBackConnector implements PipeConnector {
throws PipeException, WALPipeException {
final TSStatus status;
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
if (Objects.isNull(insertNode)) {
status =
PipeDataNodeAgent.receiver()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java
index 19c3b0ca996..6860847baf1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java
@@ -25,7 +25,7 @@ import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPend
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeHeartbeatEventMetrics;
import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.pipe.api.event.Event;
@@ -43,7 +43,6 @@ public class PipeHeartbeatEvent extends EnrichedEvent {
private final String dataRegionId;
private long timePublished;
- private long timeAssigned;
private long timeProcessed;
private long timeTransferred;
@@ -83,7 +82,7 @@ public class PipeHeartbeatEvent extends EnrichedEvent {
@Override
public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
if (Objects.nonNull(pipeName)) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.increaseHeartbeatEventCount(pipeName, creationTime);
}
return true;
@@ -94,7 +93,7 @@ public class PipeHeartbeatEvent extends EnrichedEvent {
// PipeName == null indicates that the event is the raw event at disruptor,
// not the event copied and passed to the extractor
if (Objects.nonNull(pipeName)) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.decreaseHeartbeatEventCount(pipeName, creationTime);
if (shouldPrintMessage && LOGGER.isDebugEnabled()) {
LOGGER.debug(this.toString());
@@ -151,22 +150,12 @@ public class PipeHeartbeatEvent extends EnrichedEvent {
}
}
- public void onAssigned() {
- if (shouldPrintMessage) {
- timeAssigned = System.currentTimeMillis();
- if (timePublished != 0) {
- PipeHeartbeatEventMetrics.getInstance()
- .recordPublishedToAssignedTime(timeAssigned - timePublished);
- }
- }
- }
-
public void onProcessed() {
if (shouldPrintMessage) {
timeProcessed = System.currentTimeMillis();
- if (timeAssigned != 0) {
+ if (timePublished != 0) {
PipeHeartbeatEventMetrics.getInstance()
- .recordAssignedToProcessedTime(timeProcessed - timeAssigned);
+ .recordAssignedToProcessedTime(timeProcessed - timePublished);
}
}
}
@@ -223,13 +212,9 @@ public class PipeHeartbeatEvent extends EnrichedEvent {
(timePublished != 0)
? DateTimeUtils.convertLongToDate(timePublished, "ms")
: unknownMessage;
- final String publishedToAssignedMessage =
- (timeAssigned != 0 && timePublished != 0)
- ? (timeAssigned - timePublished) + "ms"
- : unknownMessage;
final String assignedToProcessedMessage =
- (timeProcessed != 0 && timeAssigned != 0)
- ? (timeProcessed - timeAssigned) + "ms"
+ (timeProcessed != 0 && timePublished != 0)
+ ? (timeProcessed - timePublished) + "ms"
: unknownMessage;
final String processedToTransferredMessage =
(timeTransferred != 0 && timeProcessed != 0)
@@ -243,11 +228,11 @@ public class PipeHeartbeatEvent extends EnrichedEvent {
final String disruptorSizeMessage = Integer.toString(disruptorSize);
final String extractorQueueTabletSizeMessage =
- timeAssigned != 0 ? Integer.toString(extractorQueueTabletSize) : unknownMessage;
+ timePublished != 0 ? Integer.toString(extractorQueueTabletSize) : unknownMessage;
final String extractorQueueTsFileSizeMessage =
- timeAssigned != 0 ? Integer.toString(extractorQueueTsFileSize) : unknownMessage;
+ timePublished != 0 ? Integer.toString(extractorQueueTsFileSize) : unknownMessage;
final String extractorQueueSizeMessage =
- timeAssigned != 0 ? Integer.toString(extractorQueueSize) : unknownMessage;
+ timePublished != 0 ? Integer.toString(extractorQueueSize) : unknownMessage;
final String connectorQueueTabletSizeMessage =
timeProcessed != 0 ? Integer.toString(connectorQueueTabletSize) : unknownMessage;
@@ -263,9 +248,7 @@ public class PipeHeartbeatEvent extends EnrichedEvent {
+ dataRegionId
+ ", startTime="
+ startTimeMessage
- + ", publishedToAssigned="
- + publishedToAssignedMessage
- + ", assignedToProcessed="
+ + ", publishedToProcessed="
+ assignedToProcessedMessage
+ ", processedToTransferred="
+ processedToTransferredMessage
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
index e7814916e5d..6951a02e6b0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
@@ -28,8 +28,9 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager.PipeEventResource;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.ReferenceTrackableEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
@@ -37,8 +38,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryPosition;
import org.apache.iotdb.pipe.api.access.Row;
import org.apache.iotdb.pipe.api.collector.RowCollector;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
@@ -69,52 +68,26 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
LoggerFactory.getLogger(PipeInsertNodeTabletInsertionEvent.class);
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(PipeInsertNodeTabletInsertionEvent.class)
- + RamUsageEstimator.shallowSizeOfInstance(WALEntryHandler.class)
- + RamUsageEstimator.shallowSizeOfInstance(WALEntryPosition.class)
+ RamUsageEstimator.shallowSizeOfInstance(AtomicInteger.class)
+ RamUsageEstimator.shallowSizeOfInstance(AtomicBoolean.class);
- private final WALEntryHandler walEntryHandler;
- private final boolean isAligned;
- private final boolean isGeneratedByPipe;
-
private final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock;
private volatile List<Tablet> tablets;
private List<TabletInsertionDataContainer> dataContainers;
- private final PartialPath devicePath;
+ private InsertNode insertNode;
private ProgressIndex progressIndex;
private long extractTime = 0;
- public PipeInsertNodeTabletInsertionEvent(
- final WALEntryHandler walEntryHandler,
- final PartialPath devicePath,
- final ProgressIndex progressIndex,
- final boolean isAligned,
- final boolean isGeneratedByPipe) {
- this(
- walEntryHandler,
- devicePath,
- progressIndex,
- isAligned,
- isGeneratedByPipe,
- null,
- 0,
- null,
- null,
- Long.MIN_VALUE,
- Long.MAX_VALUE);
+ public PipeInsertNodeTabletInsertionEvent(final InsertNode insertNode) {
+ this(insertNode, null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE);
}
private PipeInsertNodeTabletInsertionEvent(
- final WALEntryHandler walEntryHandler,
- final PartialPath devicePath,
- final ProgressIndex progressIndex,
- final boolean isAligned,
- final boolean isGeneratedByPipe,
+ final InsertNode insertNode,
final String pipeName,
final long creationTime,
final PipeTaskMeta pipeTaskMeta,
@@ -122,34 +95,24 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
final long startTime,
final long endTime) {
super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime);
- this.walEntryHandler = walEntryHandler;
// Record device path here so there's no need to get it from InsertNode cache later.
- this.devicePath = devicePath;
- this.progressIndex = progressIndex;
- this.isAligned = isAligned;
- this.isGeneratedByPipe = isGeneratedByPipe;
-
+ this.progressIndex = insertNode.getProgressIndex();
+ this.insertNode = insertNode;
this.allocatedMemoryBlock = new AtomicReference<>();
}
- public InsertNode getInsertNode() throws WALPipeException {
- return walEntryHandler.getInsertNode();
+ public InsertNode getInsertNode() {
+ return insertNode;
}
public ByteBuffer getByteBuffer() throws WALPipeException {
- return walEntryHandler.getByteBuffer();
- }
-
- // This method is a pre-determination of whether to use binary transfers.
- // If the insert node is null in cache, it means that we need to read the bytebuffer from the wal,
- // and when the pattern is default, we can transfer the bytebuffer directly without serializing or
- // deserializing
- public InsertNode getInsertNodeViaCacheIfPossible() {
- return walEntryHandler.getInsertNodeViaCacheIfPossible();
+ return insertNode.serializeToByteBuffer();
}
public String getDeviceId() {
- return Objects.nonNull(devicePath) ? devicePath.getFullPath() : null;
+ return Objects.nonNull(insertNode.getDevicePath())
+ ? insertNode.getDevicePath().getFullPath()
+ : null;
}
/////////////////////////// EnrichedEvent ///////////////////////////
@@ -158,19 +121,16 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
extractTime = System.nanoTime();
try {
- PipeDataNodeResourceManager.wal().pin(walEntryHandler);
if (Objects.nonNull(pipeName)) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.increaseInsertNodeEventCount(pipeName, creationTime);
- PipeDataNodeAgent.task().addFloatingMemoryUsageInByte(pipeName, ramBytesUsed());
+ PipeDataNodeAgent.task()
+ .addFloatingMemoryUsageInByte(pipeName, creationTime, ramBytesUsed());
}
return true;
} catch (final Exception e) {
LOGGER.warn(
- String.format(
- "Increase reference count for memTable %d error. Holder Message: %s",
- walEntryHandler.getMemTableId(), holderMessage),
- e);
+ String.format("Increase reference count error. Holder Message: %s", holderMessage), e);
return false;
}
}
@@ -178,7 +138,6 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
try {
- PipeDataNodeResourceManager.wal().unpin(walEntryHandler);
// release the containers' memory and close memory block
if (dataContainers != null) {
dataContainers.clear();
@@ -188,17 +147,16 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
return true;
} catch (final Exception e) {
LOGGER.warn(
- String.format(
- "Decrease reference count for memtable %d error. Holder Message: %s",
- walEntryHandler.getMemTableId(), holderMessage),
- e);
+ String.format("Decrease reference count error. Holder Message: %s", holderMessage), e);
return false;
} finally {
if (Objects.nonNull(pipeName)) {
- PipeDataNodeAgent.task().decreaseFloatingMemoryUsageInByte(pipeName, ramBytesUsed());
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeAgent.task()
+ .decreaseFloatingMemoryUsageInByte(pipeName, creationTime, ramBytesUsed());
+ PipeDataNodeSinglePipeMetrics.getInstance()
.decreaseInsertNodeEventCount(pipeName, creationTime, System.nanoTime() - extractTime);
}
+ insertNode = null;
}
}
@@ -221,28 +179,18 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
final long startTime,
final long endTime) {
return new PipeInsertNodeTabletInsertionEvent(
- walEntryHandler,
- devicePath,
- progressIndex,
- isAligned,
- isGeneratedByPipe,
- pipeName,
- creationTime,
- pipeTaskMeta,
- pattern,
- startTime,
- endTime);
+ insertNode, pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime);
}
@Override
public boolean isGeneratedByPipe() {
- return isGeneratedByPipe;
+ return insertNode.isGeneratedByPipe();
}
@Override
public boolean mayEventTimeOverlappedWithTimeRange() {
try {
- final InsertNode insertNode = getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = getInsertNode();
if (Objects.isNull(insertNode)) {
return true;
}
@@ -286,7 +234,7 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public boolean mayEventPathsOverlappedWithPattern() {
try {
- final InsertNode insertNode = getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = getInsertNode();
if (Objects.isNull(insertNode)) {
return true;
}
@@ -439,8 +387,8 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public String toString() {
return String.format(
- "PipeInsertNodeTabletInsertionEvent{walEntryHandler=%s, progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s, dataContainers=%s}",
- walEntryHandler, progressIndex, isAligned, isGeneratedByPipe, dataContainers)
+ "PipeInsertNodeTabletInsertionEvent{progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s, dataContainers=%s}",
+ progressIndex, insertNode.isAligned(), insertNode.isGeneratedByPipe(), dataContainers)
+ " - "
+ super.toString();
}
@@ -448,8 +396,8 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public String coreReportMessage() {
return String.format(
- "PipeInsertNodeTabletInsertionEvent{walEntryHandler=%s, progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s}",
- walEntryHandler, progressIndex, isAligned, isGeneratedByPipe)
+ "PipeInsertNodeTabletInsertionEvent{progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s}",
+ progressIndex, insertNode.isAligned(), insertNode.isGeneratedByPipe())
+ " - "
+ super.coreReportMessage();
}
@@ -462,7 +410,7 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public long ramBytesUsed() {
return INSTANCE_SIZE
- + (Objects.nonNull(devicePath) ? PartialPath.estimateSize(devicePath) : 0)
+ + (Objects.nonNull(insertNode) ? InsertNodeMemoryEstimator.sizeOf(insertNode) : 0)
+ (Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0);
}
@@ -476,28 +424,24 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
@Override
public PipeEventResource eventResourceBuilder() {
return new PipeInsertNodeTabletInsertionEventResource(
- this.isReleased, this.referenceCount, this.walEntryHandler, this.allocatedMemoryBlock);
+ this.isReleased, this.referenceCount, this.allocatedMemoryBlock);
}
private static class PipeInsertNodeTabletInsertionEventResource extends PipeEventResource {
- private final WALEntryHandler walEntryHandler;
private final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock;
private PipeInsertNodeTabletInsertionEventResource(
final AtomicBoolean isReleased,
final AtomicInteger referenceCount,
- final WALEntryHandler walEntryHandler,
final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock) {
super(isReleased, referenceCount);
- this.walEntryHandler = walEntryHandler;
this.allocatedMemoryBlock = allocatedMemoryBlock;
}
@Override
protected void finalizeResource() {
try {
- PipeDataNodeResourceManager.wal().unpin(walEntryHandler);
allocatedMemoryBlock.getAndUpdate(
memoryBlock -> {
if (Objects.nonNull(memoryBlock)) {
@@ -506,8 +450,7 @@ public class PipeInsertNodeTabletInsertionEvent extends EnrichedEvent
return null;
});
} catch (final Exception e) {
- LOGGER.warn(
- "Decrease reference count for memTable {} error.", walEntryHandler.getMemTableId(), e);
+ LOGGER.warn("Decrease reference count error.", e);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
index 95a532708e0..9f3ad7a1bdf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
@@ -28,7 +28,7 @@ import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager.Pi
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.pipe.event.ReferenceTrackableEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock;
@@ -130,7 +130,7 @@ public class PipeRawTabletInsertionEvent extends EnrichedEvent
allocatedMemoryBlock,
PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet) + INSTANCE_SIZE);
if (Objects.nonNull(pipeName)) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.increaseRawTabletEventCount(pipeName, creationTime);
}
return true;
@@ -139,7 +139,7 @@ public class PipeRawTabletInsertionEvent extends EnrichedEvent
@Override
public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
if (Objects.nonNull(pipeName)) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.decreaseRawTabletEventCount(pipeName, creationTime);
}
allocatedMemoryBlock.close();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
index ca78dd7d50a..91d38cf3361 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
@@ -19,23 +19,16 @@
package org.apache.iotdb.db.pipe.event.common.terminate;
-import org.apache.iotdb.common.rpc.thrift.TFlushReq;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTask;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
-import org.apache.iotdb.db.storageengine.StorageEngine;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicLong;
/**
* The {@link PipeTerminateEvent} is an {@link EnrichedEvent} that controls the termination of pipe,
@@ -45,40 +38,6 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class PipeTerminateEvent extends EnrichedEvent {
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeTerminateEvent.class);
-
- private static final AtomicLong PROGRESS_REPORT_COUNT = new AtomicLong(0);
- private static final AtomicLong LAST_PROGRESS_REPORT_TIME = new AtomicLong(0);
-
- public static void flushDataRegionIfNeeded() {
- if (PROGRESS_REPORT_COUNT.get() > 0
- && PROGRESS_REPORT_COUNT.get()
- > PipeConfig.getInstance().getPipeFlushAfterTerminateCount()) {
- flushDataRegion();
- return;
- }
-
- if (LAST_PROGRESS_REPORT_TIME.get() > 0
- && System.currentTimeMillis() - LAST_PROGRESS_REPORT_TIME.get()
- > PipeConfig.getInstance().getPipeFlushAfterLastTerminateSeconds() * 1000L) {
- flushDataRegion();
- }
- }
-
- private static void flushDataRegion() {
- try {
- StorageEngine.getInstance().operateFlush(new TFlushReq());
- PROGRESS_REPORT_COUNT.set(0);
- LAST_PROGRESS_REPORT_TIME.set(0);
- LOGGER.info("Force flush all data regions because of last progress report time.");
- } catch (final Exception e) {
- LOGGER.warn(
- "Failed to flush all data regions, please check the error message: {}",
- e.getMessage(),
- e);
- }
- }
-
private final int dataRegionId;
public PipeTerminateEvent(
@@ -135,9 +94,6 @@ public class PipeTerminateEvent extends EnrichedEvent {
@Override
public void reportProgress() {
- PROGRESS_REPORT_COUNT.incrementAndGet();
- LAST_PROGRESS_REPORT_TIME.set(System.currentTimeMillis());
-
// To avoid deadlock
CompletableFuture.runAsync(
() -> PipeDataNodeAgent.task().markCompleted(pipeName, dataRegionId));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
new file mode 100644
index 00000000000..0f034dbdb96
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.event.common.tsfile;
+
+import org.apache.iotdb.commons.consensus.index.ProgressIndex;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
+import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import java.io.File;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class PipeCompactedTsFileInsertionEvent extends PipeTsFileInsertionEvent {
+
+ private final String dataRegionId;
+ private final Set<String> originFilePaths;
+ private final List<Long> commitIds;
+
+ public PipeCompactedTsFileInsertionEvent(
+ final CommitterKey committerKey,
+ final Set<PipeTsFileInsertionEvent> originalEvents,
+ final PipeTsFileInsertionEvent anyOfOriginalEvents,
+ final TsFileResource tsFileResource,
+ final boolean shouldReportProgress) {
+ super(
+ tsFileResource,
+ bindIsWithMod(originalEvents),
+ bindIsLoaded(originalEvents),
+ bindIsGeneratedByHistoricalExtractor(originalEvents),
+ committerKey.getPipeName(),
+ committerKey.getCreationTime(),
+ anyOfOriginalEvents.getPipeTaskMeta(),
+ anyOfOriginalEvents.getPipePattern(),
+ anyOfOriginalEvents.getStartTime(),
+ anyOfOriginalEvents.getEndTime());
+
+ this.dataRegionId = String.valueOf(committerKey.getRegionId());
+ this.originFilePaths =
+ originalEvents.stream()
+ .map(PipeTsFileInsertionEvent::getTsFile)
+ .map(File::getPath)
+ .collect(Collectors.toSet());
+ this.commitIds =
+ originalEvents.stream()
+ .map(PipeTsFileInsertionEvent::getCommitId)
+ .distinct()
+ .collect(Collectors.toList());
+
+ // init fields of EnrichedEvent
+ this.committerKey = committerKey;
+ // TODO pipe consensus: handle rebootTimes
+ isPatternParsed = bindIsPatternParsed(originalEvents);
+ isTimeParsed = bindIsTimeParsed(originalEvents);
+ this.shouldReportOnCommit = shouldReportProgress;
+
+ // init fields of PipeTsFileInsertionEvent
+ flushPointCount = bindFlushPointCount(originalEvents);
+ overridingProgressIndex = bindOverridingProgressIndex(originalEvents);
+ }
+
+ private static boolean bindIsWithMod(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().anyMatch(PipeTsFileInsertionEvent::isWithMod);
+ }
+
+ private static boolean bindIsLoaded(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().anyMatch(PipeTsFileInsertionEvent::isLoaded);
+ }
+
+ private static boolean bindIsGeneratedByHistoricalExtractor(
+ Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream()
+ .anyMatch(PipeTsFileInsertionEvent::isGeneratedByHistoricalExtractor);
+ }
+
+ private static boolean bindIsTimeParsed(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().noneMatch(EnrichedEvent::shouldParseTime);
+ }
+
+ private static boolean bindIsPatternParsed(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().noneMatch(EnrichedEvent::shouldParsePattern);
+ }
+
+ private static long bindFlushPointCount(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream()
+ .mapToLong(
+ e ->
+ e.getFlushPointCount() == TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET
+ ? 0
+ : e.getFlushPointCount())
+ .sum();
+ }
+
+ private ProgressIndex bindOverridingProgressIndex(Set<PipeTsFileInsertionEvent> originalEvents) {
+ ProgressIndex overridingProgressIndex = MinimumProgressIndex.INSTANCE;
+ for (PipeTsFileInsertionEvent originalEvent : originalEvents) {
+ if (originalEvent.overridingProgressIndex != null) {
+ overridingProgressIndex =
+ overridingProgressIndex.updateToMinimumEqualOrIsAfterProgressIndex(
+ originalEvent.overridingProgressIndex);
+ }
+ }
+ return overridingProgressIndex != null
+ && !overridingProgressIndex.equals(MinimumProgressIndex.INSTANCE)
+ ? overridingProgressIndex
+ : null;
+ }
+
+ @Override
+ public int getRebootTimes() {
+ throw new UnsupportedOperationException(
+ "PipeCompactedTsFileInsertionEvent does not support getRebootTimes.");
+ }
+
+ @Override
+ public boolean hasMultipleCommitIds() {
+ return true;
+ }
+
+ @Override
+ public long getCommitId() {
+ // max of commitIds is used as the commit id for this event
+ return commitIds.stream()
+ .max(Long::compareTo)
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "No commit IDs found in PipeCompactedTsFileInsertionEvent."));
+ }
+
+ // return dummy events for each commit ID (except the max one)
+ @Override
+ public List<EnrichedEvent> getDummyEventsForCommitIds() {
+ return commitIds.stream()
+ .filter(commitId -> commitId != getCommitId())
+ .map(PipeCompactedTsFileInsertionDummyEvent::new)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<Long> getCommitIds() {
+ return commitIds;
+ }
+
+ @Override
+ public boolean equalsInPipeConsensus(final Object o) {
+ throw new UnsupportedOperationException(
+ "PipeCompactedTsFileInsertionEvent does not support equalsInPipeConsensus.");
+ }
+
+ @Override
+ public void eliminateProgressIndex() {
+ if (Objects.isNull(overridingProgressIndex)) {
+ for (final String originFilePath : originFilePaths) {
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(dataRegionId, originFilePath);
+ }
+ }
+ }
+
+ public class PipeCompactedTsFileInsertionDummyEvent extends EnrichedEvent {
+
+ private final long commitId;
+
+ public PipeCompactedTsFileInsertionDummyEvent(final long commitId) {
+ super(
+ PipeCompactedTsFileInsertionEvent.this.pipeName,
+ PipeCompactedTsFileInsertionEvent.this.creationTime,
+ PipeCompactedTsFileInsertionEvent.this.pipeTaskMeta,
+ null, // PipePattern is not needed for dummy event
+ Long.MIN_VALUE,
+ Long.MAX_VALUE);
+ this.commitId = commitId; // Use the commitId passed in
+ this.shouldReportOnCommit = false; // Dummy events do not report progress
+ }
+
+ @Override
+ public long getCommitId() {
+ return commitId;
+ }
+
+ @Override
+ public boolean internallyIncreaseResourceReferenceCount(String holderMessage) {
+ return true;
+ }
+
+ @Override
+ public boolean internallyDecreaseResourceReferenceCount(String holderMessage) {
+ return true;
+ }
+
+ @Override
+ public ProgressIndex getProgressIndex() {
+ return MinimumProgressIndex.INSTANCE;
+ }
+
+ @Override
+ public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+ String pipeName,
+ long creationTime,
+ PipeTaskMeta pipeTaskMeta,
+ PipePattern pattern,
+ long startTime,
+ long endTime) {
+ return null;
+ }
+
+ @Override
+ public boolean isGeneratedByPipe() {
+ return false;
+ }
+
+ @Override
+ public boolean mayEventTimeOverlappedWithTimeRange() {
+ return false;
+ }
+
+ @Override
+ public boolean mayEventPathsOverlappedWithPattern() {
+ return false;
+ }
+
+ @Override
+ public String coreReportMessage() {
+ return "PipeCompactedTsFileInsertionDummyEvent";
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index 3aa9b4a6b13..d2c443e787e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -32,7 +32,7 @@ import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.container.TsFileInsertionDataContainer;
import org.apache.iotdb.db.pipe.event.common.tsfile.container.TsFileInsertionDataContainerProvider;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
@@ -65,44 +65,31 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileInsertionEvent.class);
- private final TsFileResource resource;
- private File tsFile;
- private long extractTime = 0;
+ protected final TsFileResource resource;
+ protected File tsFile;
+ protected long extractTime = 0;
// This is true iff the modFile exists and should be transferred
- private boolean isWithMod;
- private File modFile;
+ protected boolean isWithMod;
+ protected File modFile;
- private final boolean isLoaded;
- private final boolean isGeneratedByPipe;
- private final boolean isGeneratedByPipeConsensus;
- private final boolean isGeneratedByHistoricalExtractor;
+ protected final boolean isLoaded;
+ protected final boolean isGeneratedByPipe;
+ protected final boolean isGeneratedByPipeConsensus;
+ protected final boolean isGeneratedByHistoricalExtractor;
- private final AtomicBoolean isClosed;
- private final AtomicReference<TsFileInsertionDataContainer> dataContainer;
+ protected final AtomicBoolean isClosed;
+ protected final AtomicReference<TsFileInsertionDataContainer> dataContainer;
// The point count of the TsFile. Used for metrics on PipeConsensus' receiver side.
// May be updated after it is flushed. Should be negative if not set.
- private long flushPointCount = TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET;
+ protected long flushPointCount = TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET;
- private volatile ProgressIndex overridingProgressIndex;
+ protected volatile ProgressIndex overridingProgressIndex;
- public PipeTsFileInsertionEvent(
- final TsFileResource resource,
- final boolean isLoaded,
- final boolean isGeneratedByHistoricalExtractor) {
+ public PipeTsFileInsertionEvent(final TsFileResource resource, final boolean isLoaded) {
// The modFile must be copied before the event is assigned to the listening pipes
- this(
- resource,
- true,
- isLoaded,
- isGeneratedByHistoricalExtractor,
- null,
- 0,
- null,
- null,
- Long.MIN_VALUE,
- Long.MAX_VALUE);
+ this(resource, true, isLoaded, false, null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE);
}
public PipeTsFileInsertionEvent(
@@ -117,9 +104,13 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
final long startTime,
final long endTime) {
super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime);
-
this.resource = resource;
- tsFile = resource.getTsFile();
+
+ // For events created at assigner or historical extractor, the tsFile is get from the resource
+ // For events created for source, the tsFile is inherited from the assigner, because the
+ // original tsFile may be gone, and we need to get the assigner's hard-linked tsFile to
+ // hard-link it to each pipe dir
+ this.tsFile = resource.getTsFile();
final ModificationFile modFile = resource.getModFile();
this.isWithMod = isWithMod && modFile.exists();
@@ -130,6 +121,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
this.isGeneratedByPipeConsensus = resource.isGeneratedByPipeConsensus();
this.isGeneratedByHistoricalExtractor = isGeneratedByHistoricalExtractor;
+ this.dataContainer = new AtomicReference<>(null);
+
isClosed = new AtomicBoolean(resource.isClosed());
// Register close listener if TsFile is not closed
if (!isClosed.get()) {
@@ -165,8 +158,6 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
// If the status is "closed", then the resource status is "closed", the tsFile won't be altered
// and can be sent.
isClosed.set(resource.isClosed());
-
- this.dataContainer = new AtomicReference<>(null);
}
/**
@@ -174,6 +165,10 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
* otherwise.
*/
public boolean waitForTsFileClose() throws InterruptedException {
+ if (Objects.isNull(resource)) {
+ return true;
+ }
+
if (!isClosed.get()) {
isClosed.set(resource.isClosed());
@@ -204,6 +199,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return !resource.isEmpty();
}
+ @Override
public File getTsFile() {
return tsFile;
}
@@ -226,10 +222,6 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return isLoaded;
}
- public long getFileStartTime() {
- return resource.getFileStartTime();
- }
-
/**
* Only used for metrics on PipeConsensus' receiver side. If the event is recovered after data
* node's restart, the flushPointCount can be not set. It's totally fine for the PipeConsensus'
@@ -252,9 +244,10 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
extractTime = System.nanoTime();
try {
- tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, resource);
+ tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, pipeName);
if (isWithMod) {
- modFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, null);
+ modFile =
+ PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, pipeName);
}
return true;
} catch (final Exception e) {
@@ -266,7 +259,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return false;
} finally {
if (Objects.nonNull(pipeName)) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.increaseTsFileEventCount(pipeName, creationTime);
}
}
@@ -275,9 +268,9 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
try {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName);
if (isWithMod) {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName);
}
close();
return true;
@@ -290,7 +283,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return false;
} finally {
if (Objects.nonNull(pipeName)) {
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.decreaseTsFileEventCount(pipeName, creationTime, System.nanoTime() - extractTime);
}
}
@@ -303,24 +296,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public ProgressIndex getProgressIndex() {
- try {
- if (!waitForTsFileClose()) {
- LOGGER.warn(
- "Skipping temporary TsFile {}'s progressIndex, will report MinimumProgressIndex",
- tsFile);
- return MinimumProgressIndex.INSTANCE;
- }
- if (Objects.nonNull(overridingProgressIndex)) {
- return overridingProgressIndex;
- }
- return resource.getMaxProgressIndexAfterClose();
- } catch (final InterruptedException e) {
- LOGGER.warn(
- String.format(
- "Interrupted when waiting for closing TsFile %s.", resource.getTsFilePath()));
- Thread.currentThread().interrupt();
- return MinimumProgressIndex.INSTANCE;
- }
+ return resource.getMaxProgressIndex();
}
/**
@@ -346,7 +322,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
}
public void eliminateProgressIndex() {
- if (Objects.isNull(overridingProgressIndex)) {
+ if (Objects.isNull(overridingProgressIndex) && Objects.nonNull(resource)) {
PipeTsFileEpochProgressIndexKeeper.getInstance()
.eliminateProgressIndex(resource.getDataRegionId(), resource.getTsFilePath());
}
@@ -380,16 +356,15 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public boolean mayEventTimeOverlappedWithTimeRange() {
- // If the tsFile is not closed the resource.getFileEndTime() will be Long.MIN_VALUE
- // In that case we only judge the resource.getFileStartTime() to avoid losing data
- return isClosed.get()
- ? startTime <= resource.getFileEndTime() && resource.getFileStartTime() <= endTime
- : resource.getFileStartTime() <= endTime;
+ // Notice that this is only called at realtime extraction, and the tsFile is always closed
+ // Thus we can use the end time to judge the overlap
+ return Objects.isNull(resource)
+ || startTime <= resource.getFileEndTime() && resource.getFileStartTime() <= endTime;
}
@Override
public boolean mayEventPathsOverlappedWithPattern() {
- if (!resource.isClosed()) {
+ if (Objects.isNull(resource) || !resource.isClosed()) {
return true;
}
@@ -397,7 +372,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
final Map<IDeviceID, Boolean> deviceIsAlignedMap =
PipeDataNodeResourceManager.tsfile()
.getDeviceIsAlignedMapFromCache(
- PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()),
+ PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ resource.getTsFile(), pipeName),
false);
final Set<IDeviceID> deviceSet =
Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices();
@@ -569,7 +545,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
} catch (final IOException e) {
close();
- final String errorMsg = String.format("Read TsFile %s error.", resource.getTsFilePath());
+ final String errorMsg = String.format("Read TsFile %s error.", tsFile.getPath());
LOGGER.warn(errorMsg, e);
throw new PipeException(errorMsg);
}
@@ -617,8 +593,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public String toString() {
return String.format(
- "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, isClosed=%s, dataContainer=%s}",
- resource, tsFile, isLoaded, isGeneratedByPipe, isClosed.get(), dataContainer)
+ "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, dataContainer=%s}",
+ resource, tsFile, isLoaded, isGeneratedByPipe, dataContainer)
+ " - "
+ super.toString();
}
@@ -626,8 +602,8 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
@Override
public String coreReportMessage() {
return String.format(
- "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, isClosed=%s}",
- resource, tsFile, isLoaded, isGeneratedByPipe, isClosed.get())
+ "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s}",
+ resource, tsFile, isLoaded, isGeneratedByPipe)
+ " - "
+ super.coreReportMessage();
}
@@ -644,6 +620,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
return new PipeTsFileInsertionEventResource(
this.isReleased,
this.referenceCount,
+ this.pipeName,
this.tsFile,
this.isWithMod,
this.modFile,
@@ -656,15 +633,18 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
private final boolean isWithMod;
private final File modFile;
private final AtomicReference<TsFileInsertionDataContainer> dataContainer;
+ private final String pipeName;
private PipeTsFileInsertionEventResource(
final AtomicBoolean isReleased,
final AtomicInteger referenceCount,
+ final String pipeName,
final File tsFile,
final boolean isWithMod,
final File modFile,
final AtomicReference<TsFileInsertionDataContainer> dataContainer) {
super(isReleased, referenceCount);
+ this.pipeName = pipeName;
this.tsFile = tsFile;
this.isWithMod = isWithMod;
this.modFile = modFile;
@@ -675,9 +655,9 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
protected void finalizeResource() {
try {
// decrease reference count
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName);
if (isWithMod) {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName);
}
// close data container
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
index 21ce698141d..18ef721b924 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainerProvider.java
@@ -28,7 +28,8 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.container.query.TsFileInsert
import org.apache.iotdb.db.pipe.event.common.tsfile.container.scan.TsFileInsertionScanDataContainer;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResource;
+import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileMemResource;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.PlainDeviceID;
@@ -79,8 +80,8 @@ public class TsFileInsertionDataContainerProvider {
// Use scan container to save memory
if ((double) PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes()
- / PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- > PipeTsFileResource.MEMORY_SUFFICIENT_THRESHOLD) {
+ / PipeMemoryManager.getTotalNonFloatingMemorySizeInBytes()
+ > PipeTsFileMemResource.MEMORY_SUFFICIENT_THRESHOLD) {
return new TsFileInsertionScanDataContainer(
pipeName, creationTime, tsFile, pattern, startTime, endTime, pipeTaskMeta, sourceEvent);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java
index d4a39193467..65525ffe76a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java
@@ -84,10 +84,6 @@ public class PipeRealtimeEvent extends EnrichedEvent {
return device2Measurements;
}
- public void gcSchemaInfo() {
- device2Measurements = null;
- }
-
public boolean mayExtractorUseTablets(final PipeRealtimeDataRegionExtractor extractor) {
final TsFileEpoch.State state = tsFileEpoch.getState(extractor);
return state.equals(TsFileEpoch.State.EMPTY) || state.equals(TsFileEpoch.State.USING_TABLET);
@@ -182,9 +178,7 @@ public class PipeRealtimeEvent extends EnrichedEvent {
event.shallowCopySelfAndBindPipeTaskMetaForProgressReport(
pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime),
this.tsFileEpoch,
- // device2Measurements is not used anymore, so it is not copied.
- // If null is not passed, the field will not be GCed and may cause OOM.
- null,
+ this.device2Measurements,
pipeTaskMeta,
pattern,
startTime,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
index ef9db9a4cb0..0c3bce5c399 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
@@ -28,7 +28,6 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch.TsFileEpochM
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
public class PipeRealtimeEventFactory {
@@ -37,22 +36,13 @@ public class PipeRealtimeEventFactory {
public static PipeRealtimeEvent createRealtimeEvent(
final TsFileResource resource, final boolean isLoaded) {
return TS_FILE_EPOCH_MANAGER.bindPipeTsFileInsertionEvent(
- new PipeTsFileInsertionEvent(resource, isLoaded, false), resource);
+ new PipeTsFileInsertionEvent(resource, isLoaded), resource);
}
public static PipeRealtimeEvent createRealtimeEvent(
- final WALEntryHandler walEntryHandler,
- final InsertNode insertNode,
- final TsFileResource resource) {
+ final InsertNode insertNode, final TsFileResource resource) {
return TS_FILE_EPOCH_MANAGER.bindPipeInsertNodeTabletInsertionEvent(
- new PipeInsertNodeTabletInsertionEvent(
- walEntryHandler,
- insertNode.getDevicePath(),
- insertNode.getProgressIndex(),
- insertNode.isAligned(),
- insertNode.isGeneratedByPipe()),
- insertNode,
- resource);
+ new PipeInsertNodeTabletInsertionEvent(insertNode), insertNode, resource);
}
public static PipeRealtimeEvent createRealtimeEvent(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
index 808df12dfac..3d5b6160dac 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.pipe.extractor.dataregion;
import org.apache.iotdb.commons.consensus.DataRegionId;
-import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBPipePattern;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
import org.apache.iotdb.commons.pipe.extractor.IoTDBExtractor;
@@ -34,11 +34,10 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRe
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionHybridExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionLogExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionTsFileExtractor;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
@@ -47,11 +46,12 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.apache.iotdb.pipe.api.exception.PipeException;
-import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+
import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference;
@@ -95,13 +95,12 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionExtractor.class);
- private PipeHistoricalDataRegionExtractor historicalExtractor;
+ private @Nullable PipeHistoricalDataRegionExtractor historicalExtractor;
private PipeRealtimeDataRegionExtractor realtimeExtractor;
private DataRegionWatermarkInjector watermarkInjector;
private boolean hasNoExtractionNeed = true;
- private boolean shouldExtractInsertion = false;
private boolean shouldExtractDeletion = false;
@Override
@@ -116,7 +115,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
return;
}
hasNoExtractionNeed = false;
- shouldExtractInsertion = insertionDeletionListeningOptionPair.getLeft();
shouldExtractDeletion = insertionDeletionListeningOptionPair.getRight();
if (insertionDeletionListeningOptionPair.getLeft().equals(true)
@@ -213,10 +211,22 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
EXTRACTOR_HISTORY_END_TIME_KEY);
}
- constructHistoricalExtractor();
+ if (validator
+ .getParameters()
+ .getBooleanOrDefault(SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || validator
+ .getParameters()
+ .getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE)) {
+ // Do not flush or open historical extractor when historical tsFile is disabled
+ constructHistoricalExtractor();
+ }
constructRealtimeExtractor(validator.getParameters());
- historicalExtractor.validate(validator);
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.validate(validator);
+ }
realtimeExtractor.validate(validator);
}
@@ -237,12 +247,10 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
private void constructHistoricalExtractor() {
- // Enable historical extractor by default
historicalExtractor = new PipeHistoricalDataRegionTsFileExtractor();
}
- private void constructRealtimeExtractor(final PipeParameters parameters)
- throws IllegalPathException {
+ private void constructRealtimeExtractor(final PipeParameters parameters) {
// Use heartbeat only extractor if disable realtime extractor
if (!parameters.getBooleanOrDefault(
Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
@@ -270,7 +278,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
// Use hybrid mode by default
if (!parameters.hasAnyAttributes(EXTRACTOR_REALTIME_MODE_KEY, SOURCE_REALTIME_MODE_KEY)) {
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
LOGGER.info(
"Pipe: '{}' is not set, use hybrid mode by default.", EXTRACTOR_REALTIME_MODE_KEY);
@@ -285,15 +292,12 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
case EXTRACTOR_REALTIME_MODE_HYBRID_VALUE:
case EXTRACTOR_REALTIME_MODE_LOG_VALUE:
case EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
break;
case EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionLogExtractor();
break;
default:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
if (LOGGER.isWarnEnabled()) {
LOGGER.warn(
@@ -303,28 +307,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
}
- private void checkWalEnableAndSetUncompressed(final PipeParameters parameters)
- throws IllegalPathException {
- if (Boolean.TRUE.equals(
- DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters)
- .getLeft())
- && IoTDBDescriptor.getInstance().getConfig().getWalMode().equals(WALMode.DISABLE)) {
- throw new PipeException(
- "The pipe cannot transfer realtime insertion if data region disables wal. Please set 'realtime.mode'='batch' in source parameters when enabling realtime transmission.");
- }
-
- if (!IoTDBDescriptor.getInstance()
- .getConfig()
- .getWALCompressionAlgorithm()
- .equals(CompressionType.UNCOMPRESSED)) {
- LOGGER.info(
- "The pipe prefers uncompressed wal, and may introduce certain delay in realtime insert syncing without it. Hence, we change it to uncompressed.");
- IoTDBDescriptor.getInstance()
- .getConfig()
- .setWALCompressionAlgorithm(CompressionType.UNCOMPRESSED);
- }
- }
-
@Override
public void customize(
final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration)
@@ -335,7 +317,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
super.customize(parameters, configuration);
- historicalExtractor.customize(parameters, configuration);
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.customize(parameters, configuration);
+ }
realtimeExtractor.customize(parameters, configuration);
// Set watermark injector
@@ -358,7 +342,7 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
// register metric after generating taskID
PipeDataRegionExtractorMetrics.getInstance().register(this);
PipeTsFileToTabletsMetrics.getInstance().register(this);
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance().register(this);
+ PipeDataNodeSinglePipeMetrics.getInstance().register(this);
}
@Override
@@ -372,7 +356,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
"Pipe {}@{}: Starting historical extractor {} and realtime extractor {}.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName());
super.start();
@@ -425,14 +411,18 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
// There can still be writing when tsFile events are added. If we start
// realtimeExtractor after the process, then this part of data will be lost.
realtimeExtractor.start();
- historicalExtractor.start();
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.start();
+ }
} catch (final Exception e) {
exceptionHolder.set(e);
LOGGER.warn(
"Pipe {}@{}: Start historical extractor {} and realtime extractor {} error.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName(),
e);
}
@@ -451,7 +441,7 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
Event event = null;
- if (!historicalExtractor.hasConsumedAll()) {
+ if (Objects.nonNull(historicalExtractor) && !historicalExtractor.hasConsumedAll()) {
event = historicalExtractor.supply();
} else {
if (Objects.nonNull(watermarkInjector)) {
@@ -481,32 +471,21 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
return;
}
- historicalExtractor.close();
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.close();
+ }
realtimeExtractor.close();
if (Objects.nonNull(taskID)) {
PipeDataRegionExtractorMetrics.getInstance().deregister(taskID);
}
}
- //////////////////////////// APIs provided for detecting stuck ////////////////////////////
-
- public boolean shouldExtractInsertion() {
- return shouldExtractInsertion;
- }
-
- public boolean isStreamMode() {
- return realtimeExtractor instanceof PipeRealtimeDataRegionHybridExtractor
- || realtimeExtractor instanceof PipeRealtimeDataRegionLogExtractor;
- }
-
- public boolean hasConsumedAllHistoricalTsFiles() {
- return historicalExtractor.hasConsumedAll();
- }
-
//////////////////////////// APIs provided for metric framework ////////////////////////////
public int getHistoricalTsFileInsertionEventCount() {
- return hasBeenStarted.get() ? historicalExtractor.getPendingQueueSize() : 0;
+ return hasBeenStarted.get() && Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getPendingQueueSize()
+ : 0;
}
public int getTabletInsertionEventCount() {
@@ -520,10 +499,4 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
public int getPipeHeartbeatEventCount() {
return hasBeenStarted.get() ? realtimeExtractor.getPipeHeartbeatEventCount() : 0;
}
-
- public int getEventCount() {
- return hasBeenStarted.get()
- ? (historicalExtractor.getPendingQueueSize() + realtimeExtractor.getEventCount())
- : 0;
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
index acec7573a2a..8c8a3a31412 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
@@ -57,10 +58,10 @@ import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
@@ -91,9 +92,6 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
private static final Logger LOGGER =
LoggerFactory.getLogger(PipeHistoricalDataRegionTsFileExtractor.class);
- private static final Map<Integer, Long> DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP = new HashMap<>();
- private static final long PIPE_MIN_FLUSH_INTERVAL_IN_MS = 2000;
-
private String pipeName;
private long creationTime;
@@ -214,17 +212,14 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
try {
historicalDataExtractionStartTime =
- isHistoricalExtractorEnabled
- && parameters.hasAnyAttributes(
- EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY)
+ parameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY)
? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
parameters.getStringByKeys(
EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY))
: Long.MIN_VALUE;
historicalDataExtractionEndTime =
- isHistoricalExtractorEnabled
- && parameters.hasAnyAttributes(
- EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)
+ parameters.hasAnyAttributes(EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY)
? DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone(
parameters.getStringByKeys(
EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY))
@@ -268,10 +263,6 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
startIndex = environment.getPipeTaskMeta().restoreProgressIndex();
dataRegionId = environment.getRegionId();
- synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) {
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.putIfAbsent(dataRegionId, 0L);
- }
-
pipePattern = PipePattern.parsePipePatternFromSourceParameters(parameters);
final DataRegion dataRegion =
@@ -295,34 +286,6 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// PipeHistoricalDataRegionExtractor from implementation perspective.
: environment.getCreationTime();
- // Only invoke flushDataRegionAllTsFiles() when the pipe runs in the realtime only mode.
- // realtime only mode -> (historicalDataExtractionTimeLowerBound != Long.MIN_VALUE)
- //
- // Ensure that all data in the data region is flushed to disk before extracting data.
- // This ensures the generation time of all newly generated TsFiles (realtime data) after the
- // invocation of flushDataRegionAllTsFiles() is later than the creationTime of the pipe
- // (historicalDataExtractionTimeLowerBound).
- //
- // Note that: the generation time of the TsFile is the time when the TsFile is created, not
- // the time when the data is flushed to the TsFile.
- //
- // Then we can use the generation time of the TsFile to determine whether the data in the
- // TsFile should be extracted by comparing the generation time of the TsFile with the
- // historicalDataExtractionTimeLowerBound when starting the pipe in realtime only mode.
- //
- // If we don't invoke flushDataRegionAllTsFiles() in the realtime only mode, the data generated
- // between the creation time of the pipe the time when the pipe starts will be lost.
- if (historicalDataExtractionTimeLowerBound != Long.MIN_VALUE) {
- synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) {
- final long lastFlushedByPipeTime =
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.get(dataRegionId);
- if (System.currentTimeMillis() - lastFlushedByPipeTime >= PIPE_MIN_FLUSH_INTERVAL_IN_MS) {
- flushDataRegionAllTsFiles();
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.replace(dataRegionId, System.currentTimeMillis());
- }
- }
- }
-
shouldTransferModFile =
parameters.getBooleanOrDefault(
Arrays.asList(SOURCE_MODS_ENABLE_KEY, EXTRACTOR_MODS_ENABLE_KEY),
@@ -345,24 +308,9 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
}
}
- private void flushDataRegionAllTsFiles() {
- final DataRegion dataRegion =
- StorageEngine.getInstance().getDataRegion(new DataRegionId(dataRegionId));
- if (Objects.isNull(dataRegion)) {
- return;
- }
-
- dataRegion.writeLock("Pipe: create historical TsFile extractor");
- try {
- dataRegion.syncCloseAllWorkingTsFileProcessors();
- } finally {
- dataRegion.writeUnlock();
- }
- }
-
@Override
public synchronized void start() {
- if (!shouldExtractInsertion) {
+ if (!shouldExtractInsertion || !isHistoricalExtractorEnabled) {
hasBeenStarted = true;
return;
}
@@ -394,48 +342,29 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// consensus pipe, and the lastFlushed timestamp is not updated here.
if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) {
dataRegion.syncCloseAllWorkingTsFileProcessors();
- LOGGER.info(
- "Pipe {}@{}: finish to flush data region, took {} ms",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - startHistoricalExtractionTime);
} else {
- synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) {
- final long lastFlushedByPipeTime =
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.get(dataRegionId);
- if (System.currentTimeMillis() - lastFlushedByPipeTime >= PIPE_MIN_FLUSH_INTERVAL_IN_MS) {
- dataRegion.syncCloseAllWorkingTsFileProcessors();
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.replace(
- dataRegionId, System.currentTimeMillis());
- LOGGER.info(
- "Pipe {}@{}: finish to flush data region, took {} ms",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - startHistoricalExtractionTime);
- } else {
- LOGGER.info(
- "Pipe {}@{}: skip to flush data region, last flushed time {} ms ago",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - lastFlushedByPipeTime);
- }
- }
+ dataRegion.asyncCloseAllWorkingTsFileProcessors();
}
+ LOGGER.info(
+ "Pipe {}@{}: finish to flush data region, took {} ms",
+ pipeName,
+ dataRegionId,
+ System.currentTimeMillis() - startHistoricalExtractionTime);
final TsFileManager tsFileManager = dataRegion.getTsFileManager();
tsFileManager.readLock();
try {
final int originalSequenceTsFileCount = tsFileManager.size(true);
- final int originalUnsequenceTsFileCount = tsFileManager.size(false);
+ final int originalUnSequenceTsFileCount = tsFileManager.size(false);
final List<TsFileResource> resourceList =
- new ArrayList<>(originalSequenceTsFileCount + originalUnsequenceTsFileCount);
+ new ArrayList<>(originalSequenceTsFileCount + originalUnSequenceTsFileCount);
LOGGER.info(
"Pipe {}@{}: start to extract historical TsFile, original sequence file count {}, "
- + "original unsequence file count {}, start progress index {}",
+ + "original unSequence file count {}, start progress index {}",
pipeName,
dataRegionId,
originalSequenceTsFileCount,
- originalUnsequenceTsFileCount,
+ originalUnSequenceTsFileCount,
startIndex);
final Collection<TsFileResource> sequenceTsFileResources =
@@ -445,9 +374,13 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// Some resource is marked as deleted but not removed from the list.
!resource.isDeleted()
&& (
- // Some resource may not be closed due to the control of
- // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+ // If the tsFile is not already marked closing, it is not captured by
+ // the pipe realtime module. Thus, we can wait for the realtime sync
+ // module to handle this, to avoid blocking the pipe sync process.
!resource.isClosed()
+ && Optional.ofNullable(resource.getProcessor())
+ .map(TsFileProcessor::alreadyMarkedClosing)
+ .orElse(true)
|| mayTsFileContainUnprocessedData(resource)
&& isTsFileResourceOverlappedWithTimeRange(resource)
&& isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
@@ -455,22 +388,26 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
.collect(Collectors.toList());
resourceList.addAll(sequenceTsFileResources);
- final Collection<TsFileResource> unsequenceTsFileResources =
+ final Collection<TsFileResource> unSequenceTsFileResources =
tsFileManager.getTsFileList(false).stream()
.filter(
resource ->
// Some resource is marked as deleted but not removed from the list.
!resource.isDeleted()
&& (
- // Some resource may not be closed due to the control of
- // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+ // If the tsFile is not already marked closing, it is not captured by
+ // the pipe realtime module. Thus, we can wait for the realtime sync
+ // module to handle this, to avoid blocking the pipe sync process.
!resource.isClosed()
+ && Optional.ofNullable(resource.getProcessor())
+ .map(TsFileProcessor::alreadyMarkedClosing)
+ .orElse(true)
|| mayTsFileContainUnprocessedData(resource)
&& isTsFileResourceOverlappedWithTimeRange(resource)
&& isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
&& mayTsFileResourceOverlappedWithPattern(resource)))
.collect(Collectors.toList());
- resourceList.addAll(unsequenceTsFileResources);
+ resourceList.addAll(unSequenceTsFileResources);
resourceList.removeIf(
resource -> {
@@ -478,7 +415,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
// Will unpin it after the PipeTsFileInsertionEvent is created and pinned.
try {
PipeDataNodeResourceManager.tsfile()
- .pinTsFileResource(resource, shouldTransferModFile);
+ .pinTsFileResource(resource, shouldTransferModFile, pipeName);
return false;
} catch (final IOException e) {
LOGGER.warn("Pipe: failed to pin TsFileResource {}", resource.getTsFilePath(), e);
@@ -500,10 +437,10 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
dataRegionId,
sequenceTsFileResources.size(),
originalSequenceTsFileCount,
- unsequenceTsFileResources.size(),
- originalUnsequenceTsFileCount,
+ unSequenceTsFileResources.size(),
+ originalUnSequenceTsFileCount,
resourceList.size(),
- originalSequenceTsFileCount + originalUnsequenceTsFileCount,
+ originalSequenceTsFileCount + originalUnSequenceTsFileCount,
System.currentTimeMillis() - startHistoricalExtractionTime);
} finally {
tsFileManager.readUnlock();
@@ -522,8 +459,8 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
if (startIndex instanceof StateProgressIndex) {
startIndex = ((StateProgressIndex) startIndex).getInnerProgressIndex();
}
- return !startIndex.isAfter(resource.getMaxProgressIndexAfterClose())
- && !startIndex.equals(resource.getMaxProgressIndexAfterClose());
+ return !startIndex.isAfter(resource.getMaxProgressIndex())
+ && !startIndex.equals(resource.getMaxProgressIndex());
}
private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) {
@@ -532,7 +469,8 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
final Map<IDeviceID, Boolean> deviceIsAlignedMap =
PipeDataNodeResourceManager.tsfile()
.getDeviceIsAlignedMapFromCache(
- PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()),
+ PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ resource.getTsFile(), pipeName),
false);
deviceSet =
Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices();
@@ -591,6 +529,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
}
final TsFileResource resource = pendingQueue.poll();
+
if (resource == null) {
final PipeTerminateEvent terminateEvent =
new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, dataRegionId);
@@ -639,7 +578,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
return isReferenceCountIncreased ? event : null;
} finally {
try {
- PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource);
+ PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName);
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after creating event, original path: {}",
@@ -669,7 +608,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
pendingQueue.forEach(
resource -> {
try {
- PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource);
+ PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName);
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after dropping pipe, original path: {}",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
index c7a778357a4..ad8b6c76846 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
@@ -20,6 +20,8 @@
package org.apache.iotdb.db.pipe.extractor.dataregion.realtime;
import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.consensus.index.impl.SegmentProgressIndex;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
@@ -35,6 +37,7 @@ import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeTimePartitionListener;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.matcher.CachedSchemaPatternMatcher;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
@@ -200,6 +203,10 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
dataRegionId = String.valueOf(environment.getRegionId());
pipeTaskMeta = environment.getPipeTaskMeta();
+ if (pipeTaskMeta.getProgressIndex() instanceof MinimumProgressIndex) {
+ pipeTaskMeta.updateProgressIndex(new SegmentProgressIndex());
+ }
+
// Metrics related to TsFileEpoch are managed in PipeExtractorMetrics. These metrics are
// indexed by the taskID of IoTDBDataRegionExtractor. To avoid PipeRealtimeDataRegionExtractor
// holding a reference to IoTDBDataRegionExtractor, the taskID should be constructed to
@@ -312,29 +319,7 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
}
}
- // 1. Check if time parsing is necessary. If not, it means that the timestamps of the data
- // contained in this event are definitely within the time range [start time, end time].
- // 2. Check if the event's data timestamps may intersect with the time range. If not, it means
- // that the data timestamps of this event are definitely not within the time range.
- // 3. Check if pattern parsing is necessary. If not, it means that the paths of the data
- // contained in this event are definitely covered by the pattern.
- // 4. Check if the event's data paths may intersect with the pattern. If not, it means that the
- // data of this event is definitely not overlapped with the pattern.
- if ((!event.shouldParseTime() || event.getEvent().mayEventTimeOverlappedWithTimeRange())
- && (!event.shouldParsePattern() || event.getEvent().mayEventPathsOverlappedWithPattern())) {
- if (sloppyTimeRange) {
- // only skip parsing time for events whose data timestamps may intersect with the time range
- event.skipParsingTime();
- }
- if (sloppyPattern) {
- // only skip parsing pattern for events whose data paths may intersect with the pattern
- event.skipParsingPattern();
- }
-
- doExtract(event);
- } else {
- event.decreaseReferenceCount(PipeRealtimeDataRegionExtractor.class.getName(), false);
- }
+ doExtract(event);
synchronized (isClosed) {
if (isClosed.get()) {
@@ -395,6 +380,67 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
}
}
+ @Override
+ public Event supply() {
+ PipeRealtimeEvent realtimeEvent = getNextRealtimeEvent();
+
+ while (realtimeEvent != null) {
+ final Event suppliedEvent = doSupply(realtimeEvent);
+
+ realtimeEvent.decreaseReferenceCount(PipeRealtimeDataRegionExtractor.class.getName(), false);
+
+ if (suppliedEvent != null) {
+ return suppliedEvent;
+ }
+
+ realtimeEvent = getNextRealtimeEvent();
+ }
+
+ // means the pending queue is empty.
+ return null;
+ }
+
+ private PipeRealtimeEvent getNextRealtimeEvent() {
+ PipeRealtimeEvent realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
+
+ while (realtimeEvent != null
+ && (!CachedSchemaPatternMatcher.match(realtimeEvent, this)
+ || !coarseFilterEvent(realtimeEvent))) {
+ realtimeEvent.decreaseReferenceCount(
+ PipeRealtimeDataRegionTsFileExtractor.class.getName(), false);
+ realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
+ }
+
+ return realtimeEvent;
+ }
+
+ // This may require some time thus we leave it for processor thread instead of writing thread
+ private boolean coarseFilterEvent(final PipeRealtimeEvent event) {
+ // 1. Check if time parsing is necessary. If not, it means that the timestamps of the data
+ // contained in this event are definitely within the time range [start time, end time].
+ // 2. Check if the event's data timestamps may intersect with the time range. If not, it means
+ // that the data timestamps of this event are definitely not within the time range.
+ // 3. Check if pattern parsing is necessary. If not, it means that the paths of the data
+ // contained in this event are definitely covered by the pattern.
+ // 4. Check if the event's data paths may intersect with the pattern. If not, it means that the
+ // data of this event is definitely not overlapped with the pattern.
+ if ((!event.shouldParseTime() || event.getEvent().mayEventTimeOverlappedWithTimeRange())
+ && (!event.shouldParsePattern() || event.getEvent().mayEventPathsOverlappedWithPattern())) {
+ if (sloppyTimeRange) {
+ // only skip parsing time for events whose data timestamps may intersect with the time range
+ event.skipParsingTime();
+ }
+ if (sloppyPattern) {
+ // only skip parsing pattern for events whose data paths may intersect with the pattern
+ event.skipParsingPattern();
+ }
+ return true;
+ }
+ return false;
+ }
+
+ protected abstract Event doSupply(final PipeRealtimeEvent realtimeEvent);
+
protected Event supplyHeartbeat(final PipeRealtimeEvent event) {
if (event.increaseReferenceCount(PipeRealtimeDataRegionExtractor.class.getName())) {
return event.getEvent();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHeartbeatExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHeartbeatExtractor.java
index 1df62eecc92..7ebaa673d5d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHeartbeatExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHeartbeatExtractor.java
@@ -27,29 +27,13 @@ import org.apache.iotdb.pipe.api.event.Event;
public class PipeRealtimeDataRegionHeartbeatExtractor extends PipeRealtimeDataRegionExtractor {
@Override
- public Event supply() {
- PipeRealtimeEvent realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
-
- while (realtimeEvent != null) {
- Event suppliedEvent = null;
-
- // only supply PipeHeartbeatEvent
- if (realtimeEvent.getEvent() instanceof PipeHeartbeatEvent) {
- suppliedEvent = supplyHeartbeat(realtimeEvent);
- } else if (realtimeEvent.getEvent() instanceof ProgressReportEvent) {
- suppliedEvent = supplyDirectly(realtimeEvent);
- }
-
- realtimeEvent.decreaseReferenceCount(
- PipeRealtimeDataRegionHeartbeatExtractor.class.getName(), false);
-
- if (suppliedEvent != null) {
- return suppliedEvent;
- }
-
- realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
+ protected Event doSupply(PipeRealtimeEvent realtimeEvent) {
+ // only supply PipeHeartbeatEvent
+ if (realtimeEvent.getEvent() instanceof PipeHeartbeatEvent) {
+ return supplyHeartbeat(realtimeEvent);
+ } else if (realtimeEvent.getEvent() instanceof ProgressReportEvent) {
+ return supplyDirectly(realtimeEvent);
}
-
return null;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
index c9d16163fc8..5e086f7ffd5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
@@ -31,12 +30,9 @@ import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch.TsFileEpoch;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
-import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
@@ -145,6 +141,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
}
private void extractTsFileInsertion(final PipeRealtimeEvent event) {
+ // Notice that, if the tsFile is partially extracted because the pipe is not opened before, the
+ // former data won't be extracted
event
.getTsFileEpoch()
.migrateState(
@@ -152,34 +150,13 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
state -> {
switch (state) {
case EMPTY:
+ return ((PipeTsFileInsertionEvent) event.getEvent()).isLoaded()
+ ? TsFileEpoch.State.USING_TSFILE
+ : TsFileEpoch.State.USING_TABLET;
+ case USING_TABLET:
+ return TsFileEpoch.State.USING_TABLET;
case USING_TSFILE:
return TsFileEpoch.State.USING_TSFILE;
- case USING_TABLET:
- if (((PipeTsFileInsertionEvent) event.getEvent()).getFileStartTime()
- < event.getTsFileEpoch().getInsertNodeMinTime()) {
- // Some insert nodes in the tsfile epoch are not captured by pipe, so we should
- // capture the tsfile event to make sure all data in the tsfile epoch can be
- // extracted.
- //
- // The situation can be caused by the following operations:
- // 1. PipeA: start historical data extraction with flush
- // 2. Data insertion
- // 3. PipeB: start realtime data extraction
- // 4. PipeB: start historical data extraction without flush
- // 5. Data inserted in the step2 is not captured by PipeB, and if its tsfile
- // epoch's state is USING_TABLET, the tsfile event will be ignored, which
- // will cause the data loss in the tsfile epoch.
- LOGGER.info(
- "The tsFile {}'s epoch's start time {} is smaller than the captured insertNodes' min time {}, will regard it as data loss or un-sequential, will extract the tsFile",
- ((PipeTsFileInsertionEvent) event.getEvent()).getTsFile(),
- ((PipeTsFileInsertionEvent) event.getEvent()).getFileStartTime(),
- event.getTsFileEpoch().getInsertNodeMinTime());
- return TsFileEpoch.State.USING_BOTH;
- } else {
- // All data in the tsfile epoch has been extracted in tablet mode, so we should
- // simply keep the state of the tsfile epoch and discard the tsfile event.
- return TsFileEpoch.State.USING_TABLET;
- }
case USING_BOTH:
default:
return canNotUseTabletAnyMore(event)
@@ -225,51 +202,13 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
private boolean canNotUseTabletAnyMore(final PipeRealtimeEvent event) {
// In the following 4 cases, we should not extract this tablet event. all the data
// represented by the tablet event should be carried by the following tsfile event:
- // 0. If the remaining insert event count is too large, we need to reduce the accumulated
- // tablets.
- // 1. If Wal size > maximum size of wal buffer,
// the write operation will be throttled, so we should not extract any more tablet events.
- // 2. The shallow memory usage of the insert node has reached the dangerous threshold.
- // 3. Deprecated logics (unused by default)
- return mayRemainingInsertNodeEventExceedLimit(event)
- || mayWalSizeReachThrottleThreshold(event)
- || mayInsertNodeMemoryReachDangerousThreshold(event)
+ // 1. The shallow memory usage of the insert node has reached the dangerous threshold.
+ // 2. Deprecated logics (unused by default)
+ return mayInsertNodeMemoryReachDangerousThreshold(event)
|| canNotUseTabletAnymoreDeprecated(event);
}
- private boolean mayRemainingInsertNodeEventExceedLimit(final PipeRealtimeEvent event) {
- final boolean mayRemainingInsertEventExceedLimit =
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
- .mayRemainingInsertEventExceedLimit(pipeID);
- if (mayRemainingInsertEventExceedLimit && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(0): remaining insert event has reached max allowed insert event count {}",
- pipeName,
- dataRegionId,
- PipeConfig.getInstance().getPipeMaxAllowedRemainingInsertEventCountPerPipe()));
- }
- return mayRemainingInsertEventExceedLimit;
- }
-
- private boolean mayWalSizeReachThrottleThreshold(final PipeRealtimeEvent event) {
- final boolean mayWalSizeReachThrottleThreshold =
- 3 * WALManager.getInstance().getTotalDiskUsage()
- > IoTDBDescriptor.getInstance().getConfig().getThrottleThreshold();
- if (mayWalSizeReachThrottleThreshold && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(1): Wal size {} has reached throttle threshold {}",
- pipeName,
- dataRegionId,
- WALManager.getInstance().getTotalDiskUsage(),
- IoTDBDescriptor.getInstance().getConfig().getThrottleThreshold() / 3.0d));
- }
- return mayWalSizeReachThrottleThreshold;
- }
-
private boolean mayInsertNodeMemoryReachDangerousThreshold(final PipeRealtimeEvent event) {
final long floatingMemoryUsageInByte =
PipeDataNodeAgent.task().getFloatingMemoryUsageInByte(pipeName);
@@ -277,16 +216,16 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
final long totalFloatingMemorySizeInBytes =
PipeMemoryManager.getTotalFloatingMemorySizeInBytes();
final boolean mayInsertNodeMemoryReachDangerousThreshold =
- 3 * floatingMemoryUsageInByte * pipeCount >= 2 * totalFloatingMemorySizeInBytes;
+ floatingMemoryUsageInByte * pipeCount >= totalFloatingMemorySizeInBytes;
if (mayInsertNodeMemoryReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
logByLogManager(
l ->
l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(2): The shallow memory usage of the insert node {} has reached the dangerous threshold {}",
+ "Pipe task {}@{} canNotUseTabletAnyMore(2): The memory usage of the insert node {} has reached the dangerous threshold {}",
pipeName,
dataRegionId,
floatingMemoryUsageInByte * pipeCount,
- 2 * totalFloatingMemorySizeInBytes / 3.0d));
+ totalFloatingMemorySizeInBytes));
}
return mayInsertNodeMemoryReachDangerousThreshold;
}
@@ -305,54 +244,14 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
private boolean canNotUseTabletAnymoreDeprecated(final PipeRealtimeEvent event) {
// In the following 5 cases, we should not extract any more tablet events. all the data
// represented by the tablet events should be carried by the following tsfile event:
- // 0. If the pipe task is currently restarted.
- // 1. The number of pinned memTables has reached the dangerous threshold.
- // 2. The number of historical tsFile events to transfer has exceeded the limit.
- // 3. The number of realtime tsfile events to transfer has exceeded the limit.
- // 4. The number of linked tsFiles has reached the dangerous threshold.
- return isPipeTaskCurrentlyRestarted(event)
- || mayMemTablePinnedCountReachDangerousThreshold(event)
- || isHistoricalTsFileEventCountExceededLimit(event)
+ // 1. The number of historical tsFile events to transfer has exceeded the limit.
+ // 2. The number of realtime tsfile events to transfer has exceeded the limit.
+ // 3. The number of linked tsFiles has reached the dangerous threshold.
+ return isHistoricalTsFileEventCountExceededLimit(event)
|| isRealtimeTsFileEventCountExceededLimit(event)
|| mayTsFileLinkedCountReachDangerousThreshold(event);
}
- private boolean isPipeTaskCurrentlyRestarted(final PipeRealtimeEvent event) {
- if (!PipeConfig.getInstance().isPipeEpochKeepTsFileAfterStuckRestartEnabled()) {
- return false;
- }
-
- final boolean isPipeTaskCurrentlyRestarted =
- PipeDataNodeAgent.task().isPipeTaskCurrentlyRestarted(pipeName);
- if (isPipeTaskCurrentlyRestarted && event.mayExtractorUseTablets(this)) {
- LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(0): Pipe task is currently restarted",
- pipeName,
- dataRegionId);
- }
- return isPipeTaskCurrentlyRestarted;
- }
-
- private boolean mayMemTablePinnedCountReachDangerousThreshold(final PipeRealtimeEvent event) {
- if (PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount() == Integer.MAX_VALUE) {
- return false;
- }
- final boolean mayMemTablePinnedCountReachDangerousThreshold =
- PipeDataNodeResourceManager.wal().getPinnedWalCount()
- >= PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber();
- if (mayMemTablePinnedCountReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
- LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(1): The number of pinned memTables {} has reached the dangerous threshold {}",
- pipeName,
- dataRegionId,
- PipeDataNodeResourceManager.wal().getPinnedWalCount(),
- PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber());
- }
- return mayMemTablePinnedCountReachDangerousThreshold;
- }
-
private boolean isHistoricalTsFileEventCountExceededLimit(final PipeRealtimeEvent event) {
if (PipeConfig.getInstance().getPipeMaxAllowedHistoricalTsFilePerDataRegion()
== Integer.MAX_VALUE) {
@@ -399,56 +298,38 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
return false;
}
final boolean mayTsFileLinkedCountReachDangerousThreshold =
- PipeDataNodeResourceManager.tsfile().getLinkedTsfileCount()
+ PipeDataNodeResourceManager.tsfile().getLinkedTsFileCount(pipeName)
>= PipeConfig.getInstance().getPipeMaxAllowedLinkedTsFileCount();
if (mayTsFileLinkedCountReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
LOGGER.info(
"Pipe task {}@{} canNotUseTabletAnymoreDeprecated(4): The number of linked tsFiles {} has reached the dangerous threshold {}",
pipeName,
dataRegionId,
- PipeDataNodeResourceManager.tsfile().getLinkedTsfileCount(),
+ PipeDataNodeResourceManager.tsfile().getLinkedTsFileCount(pipeName),
PipeConfig.getInstance().getPipeMaxAllowedLinkedTsFileCount());
}
return mayTsFileLinkedCountReachDangerousThreshold;
}
@Override
- public Event supply() {
- PipeRealtimeEvent realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
-
- while (realtimeEvent != null) {
- final Event suppliedEvent;
-
- // Used to judge the type of the event, not directly for supplying.
- final Event eventToSupply = realtimeEvent.getEvent();
- if (eventToSupply instanceof TabletInsertionEvent) {
- suppliedEvent = supplyTabletInsertion(realtimeEvent);
- } else if (eventToSupply instanceof TsFileInsertionEvent) {
- suppliedEvent = supplyTsFileInsertion(realtimeEvent);
- } else if (eventToSupply instanceof PipeHeartbeatEvent) {
- suppliedEvent = supplyHeartbeat(realtimeEvent);
- } else if (eventToSupply instanceof PipeSchemaRegionWritePlanEvent
- || eventToSupply instanceof ProgressReportEvent) {
- suppliedEvent = supplyDirectly(realtimeEvent);
- } else {
- throw new UnsupportedOperationException(
- String.format(
- "Unsupported event type %s for hybrid realtime extractor %s to supply.",
- eventToSupply.getClass(), this));
- }
-
- realtimeEvent.decreaseReferenceCount(
- PipeRealtimeDataRegionHybridExtractor.class.getName(), false);
-
- if (suppliedEvent != null) {
- return suppliedEvent;
- }
-
- realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
+ protected Event doSupply(final PipeRealtimeEvent realtimeEvent) {
+ // Used to judge the type of the event, not directly for supplying.
+ final Event eventToSupply = realtimeEvent.getEvent();
+ if (eventToSupply instanceof TabletInsertionEvent) {
+ return supplyTabletInsertion(realtimeEvent);
+ } else if (eventToSupply instanceof TsFileInsertionEvent) {
+ return supplyTsFileInsertion(realtimeEvent);
+ } else if (eventToSupply instanceof PipeHeartbeatEvent) {
+ return supplyHeartbeat(realtimeEvent);
+ } else if (eventToSupply instanceof PipeSchemaRegionWritePlanEvent
+ || eventToSupply instanceof ProgressReportEvent) {
+ return supplyDirectly(realtimeEvent);
+ } else {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Unsupported event type %s for hybrid realtime extractor %s to supply.",
+ eventToSupply.getClass(), this));
}
-
- // Means the pending queue is empty.
- return null;
}
private Event supplyTabletInsertion(final PipeRealtimeEvent event) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
index 4b300355c80..74d5c371dce 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
@@ -82,10 +82,7 @@ public class PipeRealtimeDataRegionLogExtractor extends PipeRealtimeDataRegionEx
private void extractTsFileInsertion(PipeRealtimeEvent event) {
final PipeTsFileInsertionEvent tsFileInsertionEvent =
(PipeTsFileInsertionEvent) event.getEvent();
- if (!(tsFileInsertionEvent.isLoaded()
- // some insert nodes in the tsfile epoch are not captured by pipe
- || tsFileInsertionEvent.getFileStartTime()
- < event.getTsFileEpoch().getInsertNodeMinTime())) {
+ if (!tsFileInsertionEvent.isLoaded()) {
// All data in the tsfile epoch has been extracted in tablet mode, so we should
// simply ignore this event.
event.decreaseReferenceCount(PipeRealtimeDataRegionLogExtractor.class.getName(), false);
@@ -123,46 +120,29 @@ public class PipeRealtimeDataRegionLogExtractor extends PipeRealtimeDataRegionEx
}
@Override
- public Event supply() {
- PipeRealtimeEvent realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
-
- while (realtimeEvent != null) {
- Event suppliedEvent = null;
-
- if (realtimeEvent.getEvent() instanceof PipeHeartbeatEvent) {
- suppliedEvent = supplyHeartbeat(realtimeEvent);
- } else if (realtimeEvent.getEvent() instanceof PipeSchemaRegionWritePlanEvent
- || realtimeEvent.getEvent() instanceof ProgressReportEvent) {
- suppliedEvent = supplyDirectly(realtimeEvent);
- } else if (realtimeEvent.increaseReferenceCount(
- PipeRealtimeDataRegionLogExtractor.class.getName())) {
- suppliedEvent = realtimeEvent.getEvent();
- } else {
- // if the event's reference count can not be increased, it means the data represented by
- // this event is not reliable anymore. the data has been lost. we simply discard this event
- // and report the exception to PipeRuntimeAgent.
- final String errorMessage =
- String.format(
- "Event %s can not be supplied because "
- + "the reference count can not be increased, "
- + "the data represented by this event is lost",
- realtimeEvent.getEvent());
- LOGGER.error(errorMessage);
- PipeDataNodeAgent.runtime()
- .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
- }
-
- realtimeEvent.decreaseReferenceCount(
- PipeRealtimeDataRegionLogExtractor.class.getName(), false);
-
- if (suppliedEvent != null) {
- return suppliedEvent;
- }
-
- realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
+ protected Event doSupply(final PipeRealtimeEvent realtimeEvent) {
+ if (realtimeEvent.getEvent() instanceof PipeHeartbeatEvent) {
+ return supplyHeartbeat(realtimeEvent);
+ } else if (realtimeEvent.getEvent() instanceof PipeSchemaRegionWritePlanEvent
+ || realtimeEvent.getEvent() instanceof ProgressReportEvent) {
+ return supplyDirectly(realtimeEvent);
+ } else if (realtimeEvent.increaseReferenceCount(
+ PipeRealtimeDataRegionLogExtractor.class.getName())) {
+ return realtimeEvent.getEvent();
+ } else {
+ // if the event's reference count can not be increased, it means the data represented by
+ // this event is not reliable anymore. the data has been lost. we simply discard this event
+ // and report the exception to PipeRuntimeAgent.
+ final String errorMessage =
+ String.format(
+ "Event %s can not be supplied because "
+ + "the reference count can not be increased, "
+ + "the data represented by this event is lost",
+ realtimeEvent.getEvent());
+ LOGGER.error(errorMessage);
+ PipeDataNodeAgent.runtime()
+ .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
}
-
- // means the pending queue is empty.
return null;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
index 8072499b3da..d881ec49943 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
@@ -84,46 +84,30 @@ public class PipeRealtimeDataRegionTsFileExtractor extends PipeRealtimeDataRegio
}
@Override
- public Event supply() {
- PipeRealtimeEvent realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
-
- while (realtimeEvent != null) {
- Event suppliedEvent = null;
-
- if (realtimeEvent.getEvent() instanceof PipeHeartbeatEvent) {
- suppliedEvent = supplyHeartbeat(realtimeEvent);
- } else if (realtimeEvent.getEvent() instanceof PipeSchemaRegionWritePlanEvent
- || realtimeEvent.getEvent() instanceof ProgressReportEvent) {
- suppliedEvent = supplyDirectly(realtimeEvent);
- } else if (realtimeEvent.increaseReferenceCount(
- PipeRealtimeDataRegionTsFileExtractor.class.getName())) {
- suppliedEvent = realtimeEvent.getEvent();
- } else {
- // if the event's reference count can not be increased, it means the data represented by
- // this event is not reliable anymore. the data has been lost. we simply discard this event
- // and report the exception to PipeRuntimeAgent.
- final String errorMessage =
- String.format(
- "Event %s can not be supplied because "
- + "the reference count can not be increased, "
- + "the data represented by this event is lost",
- realtimeEvent.getEvent());
- LOGGER.error(errorMessage);
- PipeDataNodeAgent.runtime()
- .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
- }
-
- realtimeEvent.decreaseReferenceCount(
- PipeRealtimeDataRegionTsFileExtractor.class.getName(), false);
-
- if (suppliedEvent != null) {
- return suppliedEvent;
- }
-
- realtimeEvent = (PipeRealtimeEvent) pendingQueue.directPoll();
+ protected Event doSupply(final PipeRealtimeEvent realtimeEvent) {
+ if (realtimeEvent.getEvent() instanceof PipeHeartbeatEvent) {
+ return supplyHeartbeat(realtimeEvent);
+ } else if (realtimeEvent.getEvent() instanceof PipeSchemaRegionWritePlanEvent
+ || realtimeEvent.getEvent() instanceof ProgressReportEvent) {
+ return supplyDirectly(realtimeEvent);
+ } else if (realtimeEvent.increaseReferenceCount(
+ PipeRealtimeDataRegionTsFileExtractor.class.getName())) {
+ return realtimeEvent.getEvent();
+ } else {
+ // if the event's reference count can not be increased, it means the data represented by
+ // this event is not reliable anymore. the data has been lost. we simply discard this event
+ // and report the exception to PipeRuntimeAgent.
+ final String errorMessage =
+ String.format(
+ "Event %s can not be supplied because "
+ + "the reference count can not be increased, "
+ + "the data represented by this event is lost",
+ realtimeEvent.getEvent());
+ LOGGER.error(errorMessage);
+ PipeDataNodeAgent.runtime()
+ .report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
}
- // means the pending queue is empty.
return null;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
index ed91f636ac1..0ba9906fc7b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
@@ -21,171 +21,96 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
-import org.apache.iotdb.commons.pipe.metric.PipeEventCounter;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEventFactory;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionExtractor;
-import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.matcher.CachedSchemaPatternMatcher;
-import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.matcher.PipeDataRegionMatcher;
-import org.apache.iotdb.db.pipe.metric.source.PipeAssignerMetrics;
-import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
-import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.atomic.AtomicReference;
public class PipeDataRegionAssigner implements Closeable {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeDataRegionAssigner.class);
- private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
-
- /**
- * The {@link PipeDataRegionMatcher} is used to match the event with the extractor based on the
- * pattern.
- */
- private final PipeDataRegionMatcher matcher;
-
- /** The {@link DisruptorQueue} is used to assign the event to the extractor. */
- private final DisruptorQueue disruptor;
-
private final String dataRegionId;
-
- private int counter = 0;
+ protected final Set<PipeRealtimeDataRegionExtractor> extractors = new CopyOnWriteArraySet<>();
private final AtomicReference<ProgressIndex> maxProgressIndexForRealtimeEvent =
new AtomicReference<>(MinimumProgressIndex.INSTANCE);
- private final PipeEventCounter eventCounter = new PipeDataRegionEventCounter();
-
public String getDataRegionId() {
return dataRegionId;
}
public PipeDataRegionAssigner(final String dataRegionId) {
- this.matcher = new CachedSchemaPatternMatcher();
- this.disruptor = new DisruptorQueue(this::assignToExtractor, this::onAssignedHook);
this.dataRegionId = dataRegionId;
- PipeAssignerMetrics.getInstance().register(this);
}
- public void publishToAssign(final PipeRealtimeEvent event) {
- if (!event.increaseReferenceCount(PipeDataRegionAssigner.class.getName())) {
- LOGGER.warn(
- "The reference count of the realtime event {} cannot be increased, skipping it.", event);
- return;
- }
-
- final EnrichedEvent innerEvent = event.getEvent();
- eventCounter.increaseEventCount(innerEvent);
- if (innerEvent instanceof PipeHeartbeatEvent) {
- ((PipeHeartbeatEvent) innerEvent).onPublished();
+ public void assignToExtractor(final PipeRealtimeEvent event) {
+ if (event.getEvent() instanceof PipeHeartbeatEvent) {
+ ((PipeHeartbeatEvent) event.getEvent()).onPublished();
}
- // use synchronized here for completely preventing reference count leaks under extreme thread
- // scheduling when closing
- synchronized (this) {
- if (!disruptor.isClosed()) {
- disruptor.publish(event);
- } else {
- onAssignedHook(event);
- }
- }
- }
-
- private void onAssignedHook(final PipeRealtimeEvent realtimeEvent) {
- realtimeEvent.gcSchemaInfo();
- realtimeEvent.decreaseReferenceCount(PipeDataRegionAssigner.class.getName(), false);
-
- final EnrichedEvent innerEvent = realtimeEvent.getEvent();
- eventCounter.decreaseEventCount(innerEvent);
- if (innerEvent instanceof PipeHeartbeatEvent) {
- ((PipeHeartbeatEvent) innerEvent).onAssigned();
- }
- }
-
- private void assignToExtractor(
- final PipeRealtimeEvent event, final long sequence, final boolean endOfBatch) {
- if (disruptor.isClosed()) {
- return;
- }
-
- matcher
- .match(event)
- .forEach(
- extractor -> {
- if (disruptor.isClosed()) {
- return;
- }
-
- if (event.getEvent().isGeneratedByPipe() && !extractor.isForwardingPipeRequests()) {
- // The frequency of progress reports is limited by the counter, while progress
- // reports to TsFileInsertionEvent are not limited.
- if (!(event.getEvent() instanceof TsFileInsertionEvent)) {
- if (counter < PIPE_CONFIG.getPipeNonForwardingEventsProgressReportInterval()) {
- counter++;
- return;
- }
- counter = 0;
- }
-
- final ProgressReportEvent reportEvent =
- new ProgressReportEvent(
- extractor.getPipeName(),
- extractor.getCreationTime(),
- extractor.getPipeTaskMeta(),
- extractor.getPipePattern(),
- extractor.getRealtimeDataExtractionStartTime(),
- extractor.getRealtimeDataExtractionEndTime());
- reportEvent.bindProgressIndex(event.getProgressIndex());
- if (!reportEvent.increaseReferenceCount(PipeDataRegionAssigner.class.getName())) {
- LOGGER.warn(
- "The reference count of the event {} cannot be increased, skipping it.",
- reportEvent);
- return;
- }
- extractor.extract(PipeRealtimeEventFactory.createRealtimeEvent(reportEvent));
- return;
- }
-
- final PipeRealtimeEvent copiedEvent =
- event.shallowCopySelfAndBindPipeTaskMetaForProgressReport(
- extractor.getPipeName(),
- extractor.getCreationTime(),
- extractor.getPipeTaskMeta(),
- extractor.getPipePattern(),
- extractor.getRealtimeDataExtractionStartTime(),
- extractor.getRealtimeDataExtractionEndTime());
- final EnrichedEvent innerEvent = copiedEvent.getEvent();
- if (innerEvent instanceof PipeTsFileInsertionEvent) {
- final PipeTsFileInsertionEvent tsFileInsertionEvent =
- (PipeTsFileInsertionEvent) innerEvent;
- tsFileInsertionEvent.disableMod4NonTransferPipes(
- extractor.isShouldTransferModFile());
- }
-
- if (innerEvent instanceof PipeTsFileInsertionEvent
- || innerEvent instanceof PipeInsertNodeTabletInsertionEvent) {
- bindOrUpdateProgressIndexForRealtimeEvent(copiedEvent);
- }
-
- if (!copiedEvent.increaseReferenceCount(PipeDataRegionAssigner.class.getName())) {
- LOGGER.warn(
- "The reference count of the event {} cannot be increased, skipping it.",
- copiedEvent);
- return;
- }
- extractor.extract(copiedEvent);
- });
+ extractors.forEach(
+ extractor -> {
+ if (event.getEvent().isGeneratedByPipe() && !extractor.isForwardingPipeRequests()) {
+ final ProgressReportEvent reportEvent =
+ new ProgressReportEvent(
+ extractor.getPipeName(),
+ extractor.getCreationTime(),
+ extractor.getPipeTaskMeta(),
+ extractor.getPipePattern(),
+ extractor.getRealtimeDataExtractionStartTime(),
+ extractor.getRealtimeDataExtractionEndTime());
+ reportEvent.bindProgressIndex(event.getProgressIndex());
+ if (!reportEvent.increaseReferenceCount(PipeDataRegionAssigner.class.getName())) {
+ LOGGER.warn(
+ "The reference count of the event {} cannot be increased, skipping it.",
+ reportEvent);
+ return;
+ }
+ extractor.extract(PipeRealtimeEventFactory.createRealtimeEvent(reportEvent));
+ return;
+ }
+
+ final PipeRealtimeEvent copiedEvent =
+ event.shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+ extractor.getPipeName(),
+ extractor.getCreationTime(),
+ extractor.getPipeTaskMeta(),
+ extractor.getPipePattern(),
+ extractor.getRealtimeDataExtractionStartTime(),
+ extractor.getRealtimeDataExtractionEndTime());
+ final EnrichedEvent innerEvent = copiedEvent.getEvent();
+ if (innerEvent instanceof PipeTsFileInsertionEvent) {
+ final PipeTsFileInsertionEvent tsFileInsertionEvent =
+ (PipeTsFileInsertionEvent) innerEvent;
+ tsFileInsertionEvent.disableMod4NonTransferPipes(extractor.isShouldTransferModFile());
+ }
+
+ if (innerEvent instanceof PipeTsFileInsertionEvent
+ || innerEvent instanceof PipeInsertNodeTabletInsertionEvent) {
+ bindOrUpdateProgressIndexForRealtimeEvent(copiedEvent);
+ }
+
+ if (!copiedEvent.increaseReferenceCount(PipeDataRegionAssigner.class.getName())) {
+ LOGGER.warn(
+ "The reference count of the event {} cannot be increased, skipping it.",
+ copiedEvent);
+ return;
+ }
+ extractor.extract(copiedEvent);
+ });
}
private void bindOrUpdateProgressIndexForRealtimeEvent(final PipeRealtimeEvent event) {
@@ -217,45 +142,26 @@ public class PipeDataRegionAssigner implements Closeable {
}
public void startAssignTo(final PipeRealtimeDataRegionExtractor extractor) {
- matcher.register(extractor);
+ extractors.add(extractor);
}
public void stopAssignTo(final PipeRealtimeDataRegionExtractor extractor) {
- matcher.deregister(extractor);
+ extractors.remove(extractor);
}
public boolean notMoreExtractorNeededToBeAssigned() {
- return matcher.getRegisterCount() == 0;
+ return extractors.isEmpty();
}
- /**
- * Clear the matcher and disruptor. The method {@link PipeDataRegionAssigner#publishToAssign}
- * should not be used after calling this method.
- */
@Override
// use synchronized here for completely preventing reference count leaks under extreme thread
// scheduling when closing
public synchronized void close() {
- PipeAssignerMetrics.getInstance().deregister(dataRegionId);
-
final long startTime = System.currentTimeMillis();
- disruptor.shutdown();
- matcher.clear();
+ extractors.clear();
LOGGER.info(
"Pipe: Assigner on data region {} shutdown internal disruptor within {} ms",
dataRegionId,
System.currentTimeMillis() - startTime);
}
-
- public int getTabletInsertionEventCount() {
- return eventCounter.getTabletInsertionEventCount();
- }
-
- public int getTsFileInsertionEventCount() {
- return eventCounter.getTsFileInsertionEventCount();
- }
-
- public int getPipeHeartbeatEventCount() {
- return eventCounter.getPipeHeartbeatEventCount();
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
index 452c7188dec..69a276b9d0a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeDataR
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -111,15 +110,12 @@ public class PipeInsertionDataNodeListener {
return;
}
- assigner.publishToAssign(
+ assigner.assignToExtractor(
PipeRealtimeEventFactory.createRealtimeEvent(tsFileResource, isLoaded));
}
public void listenToInsertNode(
- String dataRegionId,
- WALEntryHandler walEntryHandler,
- InsertNode insertNode,
- TsFileResource tsFileResource) {
+ String dataRegionId, InsertNode insertNode, TsFileResource tsFileResource) {
if (listenToInsertNodeExtractorCount.get() == 0) {
return;
}
@@ -131,20 +127,21 @@ public class PipeInsertionDataNodeListener {
return;
}
- assigner.publishToAssign(
- PipeRealtimeEventFactory.createRealtimeEvent(walEntryHandler, insertNode, tsFileResource));
+ assigner.assignToExtractor(
+ PipeRealtimeEventFactory.createRealtimeEvent(insertNode, tsFileResource));
}
public void listenToHeartbeat(boolean shouldPrintMessage) {
dataRegionId2Assigner.forEach(
(key, value) ->
- value.publishToAssign(
+ value.assignToExtractor(
PipeRealtimeEventFactory.createRealtimeEvent(key, shouldPrintMessage)));
}
public void listenToDeleteData(DeleteDataNode node) {
dataRegionId2Assigner.forEach(
- (key, value) -> value.publishToAssign(PipeRealtimeEventFactory.createRealtimeEvent(node)));
+ (key, value) ->
+ value.assignToExtractor(PipeRealtimeEventFactory.createRealtimeEvent(node)));
}
/////////////////////////////// singleton ///////////////////////////////
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java
index 015ece38f8b..1e4e8d4e232 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java
@@ -19,190 +19,84 @@
package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.matcher;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionExtractor;
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-public class CachedSchemaPatternMatcher implements PipeDataRegionMatcher {
+public class CachedSchemaPatternMatcher {
protected static final Logger LOGGER = LoggerFactory.getLogger(CachedSchemaPatternMatcher.class);
- protected final ReentrantReadWriteLock lock;
-
- protected final Set<PipeRealtimeDataRegionExtractor> extractors;
- protected final Cache<String, Set<PipeRealtimeDataRegionExtractor>> deviceToExtractorsCache;
-
- public CachedSchemaPatternMatcher() {
- this.lock = new ReentrantReadWriteLock();
- // Should be thread-safe because the extractors will be returned by {@link #match} and
- // iterated by {@link #assignToExtractor}, at the same time the extractors may be added or
- // removed by {@link #register} and {@link #deregister}.
- this.extractors = new CopyOnWriteArraySet<>();
- this.deviceToExtractorsCache =
- Caffeine.newBuilder()
- .maximumSize(PipeConfig.getInstance().getPipeExtractorMatcherCacheSize())
- .build();
- }
-
- @Override
- public void register(final PipeRealtimeDataRegionExtractor extractor) {
- lock.writeLock().lock();
- try {
- extractors.add(extractor);
- deviceToExtractorsCache.invalidateAll();
- } finally {
- lock.writeLock().unlock();
+ public static boolean match(
+ final PipeRealtimeEvent event, final PipeRealtimeDataRegionExtractor extractor) {
+ // HeartbeatEvent will be assigned to all extractors
+ if (event.getEvent() instanceof PipeHeartbeatEvent) {
+ return true;
}
- }
- @Override
- public void deregister(final PipeRealtimeDataRegionExtractor extractor) {
- lock.writeLock().lock();
- try {
- extractors.remove(extractor);
- deviceToExtractorsCache.invalidateAll();
- } finally {
- lock.writeLock().unlock();
+ // Deletion event will be assigned to extractors listened to it
+ if (event.getEvent() instanceof PipeSchemaRegionWritePlanEvent) {
+ return extractor.shouldExtractDeletion();
}
- }
- @Override
- public int getRegisterCount() {
- lock.readLock().lock();
- try {
- return extractors.size();
- } finally {
- lock.readLock().unlock();
- }
- }
-
- @Override
- public Set<PipeRealtimeDataRegionExtractor> match(final PipeRealtimeEvent event) {
- final Set<PipeRealtimeDataRegionExtractor> matchedExtractors = new HashSet<>();
-
- lock.readLock().lock();
- try {
- if (extractors.isEmpty()) {
- return matchedExtractors;
- }
-
- // HeartbeatEvent will be assigned to all extractors
- if (event.getEvent() instanceof PipeHeartbeatEvent) {
- return extractors;
- }
+ for (final Map.Entry<String, String[]> entry : event.getSchemaInfo().entrySet()) {
+ final String device = entry.getKey();
+ final String[] measurements = entry.getValue();
- // Deletion event will be assigned to extractors listened to it
- if (event.getEvent() instanceof PipeSchemaRegionWritePlanEvent) {
- return extractors.stream()
- .filter(PipeRealtimeDataRegionExtractor::shouldExtractDeletion)
- .collect(Collectors.toSet());
+ if (!filterByDevice(device, extractor)) {
+ return false;
}
- for (final Map.Entry<String, String[]> entry : event.getSchemaInfo().entrySet()) {
- final String device = entry.getKey();
- final String[] measurements = entry.getValue();
-
- // 1. try to get matched extractors from cache, if not success, match them by device
- final Set<PipeRealtimeDataRegionExtractor> extractorsFilteredByDevice =
- deviceToExtractorsCache.get(device, this::filterExtractorsByDevice);
- // this would not happen
- if (extractorsFilteredByDevice == null) {
- LOGGER.warn("Match result NPE when handle device {}", device);
- continue;
- }
-
- // 2. filter matched candidate extractors by measurements
- if (measurements.length == 0) {
- // `measurements` is empty (only in case of tsfile event). match all extractors.
- //
- // case 1: the pattern can match all measurements of the device.
- // in this case, the extractor can be matched without checking the measurements.
- //
- // case 2: the pattern may match some measurements of the device.
- // in this case, we can't get all measurements efficiently here,
- // so we just ASSUME the extractor matches and do more checks later.
- matchedExtractors.addAll(extractorsFilteredByDevice);
+ // 2. filter matched candidate extractors by measurements
+ if (measurements.length == 0) {
+ // `measurements` is empty (only in case of tsfile event). match all extractors.
+ //
+ // case 1: the pattern can match all measurements of the device.
+ // in this case, the extractor can be matched without checking the measurements.
+ //
+ // case 2: the pattern may match some measurements of the device.
+ // in this case, we can't get all measurements efficiently here,
+ // so we just ASSUME the extractor matches and do more checks later.
+ return true;
+ } else {
+ final PipePattern pattern = extractor.getPipePattern();
+ if (Objects.isNull(pattern) || pattern.isRoot() || pattern.coversDevice(device)) {
+ // The pattern can match all measurements of the device.
+ return true;
} else {
- // `measurements` is not empty (only in case of tablet event).
- // Match extractors by measurements.
- extractorsFilteredByDevice.forEach(
- extractor -> {
- final PipePattern pattern = extractor.getPipePattern();
- if (Objects.isNull(pattern) || pattern.isRoot() || pattern.coversDevice(device)) {
- // The pattern can match all measurements of the device.
- matchedExtractors.add(extractor);
- } else {
- for (final String measurement : measurements) {
- // Ignore null measurement for partial insert
- if (measurement == null) {
- continue;
- }
-
- if (pattern.matchesMeasurement(device, measurement)) {
- matchedExtractors.add(extractor);
- // There would be no more matched extractors because the measurements are
- // unique
- break;
- }
- }
- }
- });
- }
-
- if (matchedExtractors.size() == extractors.size()) {
- break;
+ for (final String measurement : measurements) {
+ // Ignore null measurement for partial insert
+ if (measurement == null) {
+ continue;
+ }
+
+ if (pattern.matchesMeasurement(device, measurement)) {
+ return true;
+ }
+ }
}
}
- } finally {
- lock.readLock().unlock();
}
- return matchedExtractors;
+ return false;
}
- protected Set<PipeRealtimeDataRegionExtractor> filterExtractorsByDevice(final String device) {
- final Set<PipeRealtimeDataRegionExtractor> filteredExtractors = new HashSet<>();
-
- for (final PipeRealtimeDataRegionExtractor extractor : extractors) {
- // Return if the extractor only extract deletion
- if (!extractor.shouldExtractInsertion()) {
- continue;
- }
-
- final PipePattern pipePattern = extractor.getPipePattern();
- if (Objects.isNull(pipePattern) || pipePattern.mayOverlapWithDevice(device)) {
- filteredExtractors.add(extractor);
- }
- }
-
- return filteredExtractors;
+ private static boolean filterByDevice(
+ final String device, final PipeRealtimeDataRegionExtractor extractor) {
+ return extractor.shouldExtractInsertion()
+ && (Objects.isNull(extractor.getPipePattern())
+ || extractor.getPipePattern().mayOverlapWithDevice(device));
}
- @Override
- public void clear() {
- lock.writeLock().lock();
- try {
- extractors.clear();
- deviceToExtractorsCache.invalidateAll();
- deviceToExtractorsCache.cleanUp();
- } finally {
- lock.writeLock().unlock();
- }
+ private CachedSchemaPatternMatcher() {
+ // Utility class
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/matcher/PipeDataRegionMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/matcher/PipeDataRegionMatcher.java
deleted file mode 100644
index 4e102a1f7cf..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/matcher/PipeDataRegionMatcher.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.matcher;
-
-import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
-import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionExtractor;
-
-import java.util.Set;
-
-public interface PipeDataRegionMatcher {
-
- /**
- * Register a extractor. If the extractor's pattern matches the event's schema info, the event
- * will be assigned to the extractor.
- */
- void register(PipeRealtimeDataRegionExtractor extractor);
-
- /** Deregister a extractor. */
- void deregister(PipeRealtimeDataRegionExtractor extractor);
-
- /** Get the number of registered extractors in this matcher. */
- int getRegisterCount();
-
- /**
- * Match the event's schema info with the registered extractors' patterns. If the event's schema
- * info matches the pattern of a extractor, the extractor will be returned.
- *
- * @param event the event to be matched
- * @return the matched extractors
- */
- Set<PipeRealtimeDataRegionExtractor> match(PipeRealtimeEvent event);
-
- /** Clear all the registered extractors and internal data structures. */
- void clear();
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java
index b1153071bab..92fc23775e3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/IoTDBSchemaRegionExtractor.java
@@ -32,7 +32,7 @@ import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionSnapshotEvent;
import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionExtractorMetrics;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
@@ -75,7 +75,7 @@ public class IoTDBSchemaRegionExtractor extends IoTDBNonDataRegionExtractor {
listenedTypeSet = SchemaRegionListeningFilter.parseListeningPlanTypeSet(parameters);
PipeSchemaRegionExtractorMetrics.getInstance().register(this);
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance().register(this);
+ PipeDataNodeSinglePipeMetrics.getInstance().register(this);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
index 3f03ce580fd..f5a48b3f6b6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
@@ -20,18 +20,16 @@
package org.apache.iotdb.db.pipe.metric;
import org.apache.iotdb.commons.pipe.metric.PipeEventCommitMetrics;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeHeartbeatEventMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
-import org.apache.iotdb.db.pipe.metric.overview.PipeWALInsertNodeCacheMetrics;
import org.apache.iotdb.db.pipe.metric.processor.PipeProcessorMetrics;
import org.apache.iotdb.db.pipe.metric.receiver.PipeDataNodeReceiverMetrics;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionConnectorMetrics;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionExtractorMetrics;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics;
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
-import org.apache.iotdb.db.pipe.metric.source.PipeAssignerMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
@@ -42,36 +40,32 @@ public class PipeDataNodeMetrics implements IMetricSet {
@Override
public void bindTo(final AbstractMetricService metricService) {
- PipeAssignerMetrics.getInstance().bindTo(metricService);
PipeDataRegionExtractorMetrics.getInstance().bindTo(metricService);
PipeProcessorMetrics.getInstance().bindTo(metricService);
PipeDataRegionConnectorMetrics.getInstance().bindTo(metricService);
PipeHeartbeatEventMetrics.getInstance().bindTo(metricService);
- PipeWALInsertNodeCacheMetrics.getInstance().bindTo(metricService);
PipeResourceMetrics.getInstance().bindTo(metricService);
PipeEventCommitMetrics.getInstance().bindTo(metricService);
PipeSchemaRegionListenerMetrics.getInstance().bindTo(metricService);
PipeSchemaRegionExtractorMetrics.getInstance().bindTo(metricService);
PipeSchemaRegionConnectorMetrics.getInstance().bindTo(metricService);
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance().bindTo(metricService);
+ PipeDataNodeSinglePipeMetrics.getInstance().bindTo(metricService);
PipeDataNodeReceiverMetrics.getInstance().bindTo(metricService);
PipeTsFileToTabletsMetrics.getInstance().bindTo(metricService);
}
@Override
public void unbindFrom(final AbstractMetricService metricService) {
- PipeAssignerMetrics.getInstance().unbindFrom(metricService);
PipeDataRegionExtractorMetrics.getInstance().unbindFrom(metricService);
PipeProcessorMetrics.getInstance().unbindFrom(metricService);
PipeDataRegionConnectorMetrics.getInstance().unbindFrom(metricService);
PipeHeartbeatEventMetrics.getInstance().unbindFrom(metricService);
- PipeWALInsertNodeCacheMetrics.getInstance().unbindFrom(metricService);
PipeResourceMetrics.getInstance().unbindFrom(metricService);
PipeEventCommitMetrics.getInstance().unbindFrom(metricService);
PipeSchemaRegionListenerMetrics.getInstance().unbindFrom(metricService);
PipeSchemaRegionExtractorMetrics.getInstance().unbindFrom(metricService);
PipeSchemaRegionConnectorMetrics.getInstance().unbindFrom(metricService);
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance().unbindFrom(metricService);
+ PipeDataNodeSinglePipeMetrics.getInstance().unbindFrom(metricService);
PipeDataNodeReceiverMetrics.getInstance().unbindFrom(metricService);
PipeTsFileToTabletsMetrics.getInstance().unbindFrom(metricService);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
index 86368acf353..87c7c6731c9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
@@ -40,7 +40,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator {
+public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator {
// Calculate from schema region extractors directly for it requires less computation
private final Set<IoTDBSchemaRegionExtractor> schemaRegionExtractors =
@@ -59,8 +59,6 @@ class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator {
private Timer insertNodeTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer tsfileTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- private final InsertNodeEMA insertNodeEventCountEMA = new InsertNodeEMA();
-
private double lastDataRegionCommitSmoothingValue = Long.MAX_VALUE;
private double lastSchemaRegionCommitSmoothingValue = Long.MAX_VALUE;
@@ -102,12 +100,23 @@ class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator {
heartbeatEventCount.decrementAndGet();
}
- double getRemainingInsertEventSmoothingCount() {
- insertNodeEventCountEMA.update(insertNodeEventCount.get());
- return insertNodeEventCountEMA.insertNodeEMAValue;
+ public long getRemainingNonHeartbeatEvents() {
+ final long remainingEvents =
+ tsfileEventCount.get()
+ + rawTabletEventCount.get()
+ + insertNodeEventCount.get()
+ + schemaRegionExtractors.stream()
+ .map(IoTDBSchemaRegionExtractor::getUnTransferredEventCount)
+ .reduce(Long::sum)
+ .orElse(0L);
+
+ // There are cases where the indicator is negative. For example, after the Pipe is restarted,
+ // the Processor SubTask is still collecting Events, resulting in a negative count. This
+ // situation cannot be avoided because the Pipe may be restarted internally.
+ return remainingEvents >= 0 ? remainingEvents : 0;
}
- long getRemainingEvents() {
+ public long getRemainingEvents() {
final long remainingEvents =
tsfileEventCount.get()
+ rawTabletEventCount.get()
@@ -131,7 +140,7 @@ class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator {
*
* @return The estimated remaining time
*/
- double getRemainingTime() {
+ public double getRemainingTime() {
final PipeRateAverage pipeRemainingTimeCommitRateAverageTime =
PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime();
@@ -266,17 +275,4 @@ class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOperator {
dataRegionCommitMeter.set(null);
schemaRegionCommitMeter.set(null);
}
-
- private static class InsertNodeEMA {
- private double insertNodeEMAValue;
-
- public void update(final double newValue) {
- final double alpha = PipeConfig.getInstance().getPipeRemainingInsertNodeCountEMAAlpha();
- if (insertNodeEMAValue == 0) {
- insertNodeEMAValue = newValue;
- } else {
- insertNodeEMAValue = alpha * newValue + (1 - alpha) * insertNodeEMAValue;
- }
- }
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
similarity index 81%
rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeMetrics.java
rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
index 354a980edfd..534958e9011 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
@@ -20,11 +20,12 @@
package org.apache.iotdb.db.pipe.metric.overview;
import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
@@ -42,15 +43,14 @@ import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
+public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
- private static final Logger LOGGER =
- LoggerFactory.getLogger(PipeDataNodeRemainingEventAndTimeMetrics.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeDataNodeSinglePipeMetrics.class);
@SuppressWarnings("java:S3077")
private volatile AbstractMetricService metricService;
- private final Map<String, PipeDataNodeRemainingEventAndTimeOperator>
+ public final Map<String, PipeDataNodeRemainingEventAndTimeOperator>
remainingEventAndTimeOperatorMap = new ConcurrentHashMap<>();
private static Histogram PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM =
@@ -104,6 +104,35 @@ public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
Tag.CREATION_TIME.toString(),
String.valueOf(operator.getCreationTime()));
+ // Resources
+ metricService.createAutoGauge(
+ Metric.PIPE_FLOATING_MEMORY_USAGE.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeAgent.task(),
+ a -> a.getFloatingMemoryUsageInByte(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.createAutoGauge(
+ Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeResourceManager.tsfile(),
+ a -> a.getLinkedTsFileCount(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.createAutoGauge(
+ Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeResourceManager.tsfile(),
+ a -> a.getTotalLinkedTsFileSize(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+
operator.setInsertNodeTransferTimer(
metricService.getOrCreateTimer(
Metric.PIPE_INSERT_NODE_EVENT_TRANSFER_TIME.toString(),
@@ -119,31 +148,6 @@ public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
operator.getPipeName()));
}
- public boolean mayRemainingInsertEventExceedLimit(final String pipeID) {
- if (Objects.isNull(metricService)) {
- return true;
- }
-
- if (remainingEventAndTimeOperatorMap.values().stream()
- .map(PipeDataNodeRemainingEventAndTimeOperator::getRemainingInsertEventSmoothingCount)
- .reduce(0d, Double::sum)
- > PipeConfig.getInstance().getPipeMaxAllowedTotalRemainingInsertEventCount()) {
- return true;
- }
-
- final PipeDataNodeRemainingEventAndTimeOperator operator =
- remainingEventAndTimeOperatorMap.get(pipeID);
- if (Objects.isNull(operator)) {
- LOGGER.warn(
- "Failed to get remaining insert event, RemainingEventAndTimeOperator({}) does not exist, will degrade anyway",
- pipeID);
- return true;
- }
-
- return operator.getRemainingInsertEventSmoothingCount()
- > PipeConfig.getInstance().getPipeMaxAllowedRemainingInsertEventCountPerPipe();
- }
-
@Override
public void unbindFrom(final AbstractMetricService metricService) {
ImmutableSet.copyOf(remainingEventAndTimeOperatorMap.keySet()).forEach(this::deregister);
@@ -185,6 +189,27 @@ public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
operator.getPipeName(),
Tag.CREATION_TIME.toString(),
String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_FLOATING_MEMORY_USAGE.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
metricService.remove(
MetricType.TIMER,
Metric.PIPE_INSERT_NODE_EVENT_TRANSFER_TIME.toString(),
@@ -205,12 +230,13 @@ public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
final String pipeID = extractor.getPipeName() + "_" + extractor.getCreationTime();
remainingEventAndTimeOperatorMap.computeIfAbsent(
pipeID,
- k ->
- new PipeDataNodeRemainingEventAndTimeOperator(
- extractor.getPipeName(), extractor.getCreationTime()));
- if (Objects.nonNull(metricService)) {
- createMetrics(pipeID);
- }
+ k -> {
+ if (Objects.nonNull(metricService)) {
+ createMetrics(pipeID);
+ }
+ return new PipeDataNodeRemainingEventAndTimeOperator(
+ extractor.getPipeName(), extractor.getCreationTime());
+ });
}
public void register(final IoTDBSchemaRegionExtractor extractor) {
@@ -219,13 +245,14 @@ public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
remainingEventAndTimeOperatorMap
.computeIfAbsent(
pipeID,
- k ->
- new PipeDataNodeRemainingEventAndTimeOperator(
- extractor.getPipeName(), extractor.getCreationTime()))
+ k -> {
+ if (Objects.nonNull(metricService)) {
+ createMetrics(pipeID);
+ }
+ return new PipeDataNodeRemainingEventAndTimeOperator(
+ extractor.getPipeName(), extractor.getCreationTime());
+ })
.register(extractor);
- if (Objects.nonNull(metricService)) {
- createMetrics(pipeID);
- }
}
public void increaseInsertNodeEventCount(final String pipeName, final long creationTime) {
@@ -381,19 +408,19 @@ public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
private static class PipeDataNodeRemainingEventAndTimeMetricsHolder {
- private static final PipeDataNodeRemainingEventAndTimeMetrics INSTANCE =
- new PipeDataNodeRemainingEventAndTimeMetrics();
+ private static final PipeDataNodeSinglePipeMetrics INSTANCE =
+ new PipeDataNodeSinglePipeMetrics();
private PipeDataNodeRemainingEventAndTimeMetricsHolder() {
// Empty constructor
}
}
- public static PipeDataNodeRemainingEventAndTimeMetrics getInstance() {
+ public static PipeDataNodeSinglePipeMetrics getInstance() {
return PipeDataNodeRemainingEventAndTimeMetricsHolder.INSTANCE;
}
- private PipeDataNodeRemainingEventAndTimeMetrics() {
+ private PipeDataNodeSinglePipeMetrics() {
PipeEventCommitManager.getInstance().setCommitRateMarker(this::markRegionCommit);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
index c9115575000..37f8eb5e26a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
@@ -22,10 +22,9 @@ package org.apache.iotdb.db.pipe.metric.overview;
import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
import org.apache.iotdb.metrics.utils.MetricLevel;
@@ -34,6 +33,7 @@ import org.apache.iotdb.metrics.utils.MetricType;
public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_USED_MEMORY = "PipeUsedMemory";
+ private static final String PIPE_USED_FLOATING_MEMORY = "PipeUsedFloatingMemory";
private static final String PIPE_TABLET_USED_MEMORY = "PipeTabletUsedMemory";
@@ -41,6 +41,8 @@ public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_TOTAL_MEMORY = "PipeTotalMemory";
+ private static final String PIPE_FLOATING_MEMORY = "PipeFloatingMemory";
+
//////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
@Override
@@ -74,22 +76,20 @@ public class PipeResourceMetrics implements IMetricSet {
o -> PipeMemoryManager.getTotalNonFloatingMemorySizeInBytes(),
Tag.NAME.toString(),
PIPE_TOTAL_MEMORY);
- // resource reference count
metricService.createAutoGauge(
- Metric.PIPE_PINNED_MEMTABLE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.wal(),
- PipeWALResourceManager::getPinnedWalCount);
- metricService.createAutoGauge(
- Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ Metric.PIPE_MEM.toString(),
MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.tsfile(),
- PipeTsFileResourceManager::getLinkedTsfileCount);
+ PipeDataNodeResourceManager.memory(),
+ o -> PipeMemoryManager.getTotalFloatingMemorySizeInBytes(),
+ Tag.NAME.toString(),
+ PIPE_FLOATING_MEMORY);
metricService.createAutoGauge(
- Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ Metric.PIPE_MEM.toString(),
MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.tsfile(),
- PipeTsFileResourceManager::getTotalLinkedTsfileSize);
+ PipeDataNodeResourceManager.memory(),
+ o -> PipeDataNodeAgent.task().getAllFloatingMemoryUsageInByte(),
+ Tag.NAME.toString(),
+ PIPE_USED_FLOATING_MEMORY);
// phantom reference count
metricService.createAutoGauge(
Metric.PIPE_PHANTOM_REFERENCE_COUNT.toString(),
@@ -115,8 +115,17 @@ public class PipeResourceMetrics implements IMetricSet {
PIPE_TS_FILE_USED_MEMORY);
metricService.remove(
MetricType.AUTO_GAUGE, Metric.PIPE_MEM.toString(), Tag.NAME.toString(), PIPE_TOTAL_MEMORY);
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_MEM.toString(),
+ Tag.NAME.toString(),
+ PIPE_FLOATING_MEMORY);
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_MEM.toString(),
+ Tag.NAME.toString(),
+ PIPE_USED_FLOATING_MEMORY);
// resource reference count
- metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_PINNED_MEMTABLE_COUNT.toString());
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_COUNT.toString());
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_SIZE.toString());
// phantom reference count
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java
index 04015fe6f82..16108b36c5f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java
@@ -104,8 +104,7 @@ public class PipeTsFileToTabletsMetrics implements IMetricSet {
public void register(final IoTDBDataRegionExtractor extractor) {
final String pipeID = extractor.getPipeName() + "_" + extractor.getCreationTime();
- pipe.add(pipeID);
- if (Objects.nonNull(metricService)) {
+ if (Objects.nonNull(metricService) && pipe.add(pipeID)) {
createMetrics(pipeID);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
deleted file mode 100644
index b2e605bf84c..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.metric.overview;
-
-import org.apache.iotdb.commons.service.metric.enums.Metric;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
-import org.apache.iotdb.metrics.AbstractMetricService;
-import org.apache.iotdb.metrics.metricsets.IMetricSet;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.MetricType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class PipeWALInsertNodeCacheMetrics implements IMetricSet {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALInsertNodeCacheMetrics.class);
-
- //////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
-
- @Override
- public void bindTo(AbstractMetricService metricService) {
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_RATE.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheHitRate);
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_COUNT.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheHitCount);
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_REQUEST_COUNT.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheRequestCount);
- }
-
- @Override
- public void unbindFrom(AbstractMetricService metricService) {
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_RATE.toString());
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_COUNT.toString());
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_REQUEST_COUNT.toString());
- }
-
- //////////////////////////// singleton ////////////////////////////
-
- private static class PipeWALInsertNodeCacheMetricsHolder {
-
- private static final PipeWALInsertNodeCacheMetrics INSTANCE =
- new PipeWALInsertNodeCacheMetrics();
-
- private PipeWALInsertNodeCacheMetricsHolder() {
- // empty constructor
- }
- }
-
- public static PipeWALInsertNodeCacheMetrics getInstance() {
- return PipeWALInsertNodeCacheMetricsHolder.INSTANCE;
- }
-
- private PipeWALInsertNodeCacheMetrics() {
- // empty constructor
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/source/PipeAssignerMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/source/PipeAssignerMetrics.java
deleted file mode 100644
index 3a35305584f..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/source/PipeAssignerMetrics.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.metric.source;
-
-import org.apache.iotdb.commons.service.metric.enums.Metric;
-import org.apache.iotdb.commons.service.metric.enums.Tag;
-import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeDataRegionAssigner;
-import org.apache.iotdb.metrics.AbstractMetricService;
-import org.apache.iotdb.metrics.metricsets.IMetricSet;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.MetricType;
-
-import com.google.common.collect.ImmutableSet;
-import org.checkerframework.checker.nullness.qual.NonNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-public class PipeAssignerMetrics implements IMetricSet {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeAssignerMetrics.class);
-
- private AbstractMetricService metricService;
-
- private final Map<String, PipeDataRegionAssigner> assignerMap = new HashMap<>();
-
- //////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
-
- @Override
- public void bindTo(AbstractMetricService metricService) {
- this.metricService = metricService;
- synchronized (this) {
- for (String dataRegionId : assignerMap.keySet()) {
- createMetrics(dataRegionId);
- }
- }
- }
-
- private void createMetrics(String dataRegionId) {
- createAutoGauge(dataRegionId);
- }
-
- private void createAutoGauge(String dataRegionId) {
- metricService.createAutoGauge(
- Metric.UNASSIGNED_HEARTBEAT_COUNT.toString(),
- MetricLevel.IMPORTANT,
- assignerMap.get(dataRegionId),
- PipeDataRegionAssigner::getPipeHeartbeatEventCount,
- Tag.REGION.toString(),
- dataRegionId);
- metricService.createAutoGauge(
- Metric.UNASSIGNED_TABLET_COUNT.toString(),
- MetricLevel.IMPORTANT,
- assignerMap.get(dataRegionId),
- PipeDataRegionAssigner::getTabletInsertionEventCount,
- Tag.REGION.toString(),
- dataRegionId);
- metricService.createAutoGauge(
- Metric.UNASSIGNED_TSFILE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- assignerMap.get(dataRegionId),
- PipeDataRegionAssigner::getTsFileInsertionEventCount,
- Tag.REGION.toString(),
- dataRegionId);
- }
-
- @Override
- public void unbindFrom(AbstractMetricService metricService) {
- ImmutableSet<String> dataRegionIds = ImmutableSet.copyOf(assignerMap.keySet());
- for (String dataRegionId : dataRegionIds) {
- deregister(dataRegionId);
- }
- if (!assignerMap.isEmpty()) {
- LOGGER.warn("Failed to unbind from pipe assigner metrics, assigner map not empty");
- }
- }
-
- private void removeMetrics(String dataRegionId) {
- removeAutoGauge(dataRegionId);
- }
-
- private void removeAutoGauge(String dataRegionId) {
- metricService.remove(
- MetricType.AUTO_GAUGE,
- Metric.UNASSIGNED_HEARTBEAT_COUNT.toString(),
- Tag.REGION.toString(),
- dataRegionId);
- metricService.remove(
- MetricType.AUTO_GAUGE,
- Metric.UNASSIGNED_TABLET_COUNT.toString(),
- Tag.REGION.toString(),
- dataRegionId);
- metricService.remove(
- MetricType.AUTO_GAUGE,
- Metric.UNASSIGNED_TSFILE_COUNT.toString(),
- Tag.REGION.toString(),
- dataRegionId);
- }
-
- //////////////////////////// register & deregister (pipe integration) ////////////////////////////
-
- public void register(@NonNull PipeDataRegionAssigner pipeDataRegionAssigner) {
- String dataRegionId = pipeDataRegionAssigner.getDataRegionId();
- synchronized (this) {
- assignerMap.putIfAbsent(dataRegionId, pipeDataRegionAssigner);
- if (Objects.nonNull(metricService)) {
- createMetrics(dataRegionId);
- }
- }
- }
-
- public void deregister(String dataRegionId) {
- synchronized (this) {
- if (!assignerMap.containsKey(dataRegionId)) {
- LOGGER.warn(
- "Failed to deregister pipe assigner metrics, PipeDataRegionAssigner({}) does not exist",
- dataRegionId);
- return;
- }
- if (Objects.nonNull(metricService)) {
- removeMetrics(dataRegionId);
- }
- assignerMap.remove(dataRegionId);
- }
- }
-
- //////////////////////////// singleton ////////////////////////////
-
- private static class PipeAssignerMetricsHolder {
-
- private static final PipeAssignerMetrics INSTANCE = new PipeAssignerMetrics();
-
- private PipeAssignerMetricsHolder() {
- // empty constructor
- }
- }
-
- public static PipeAssignerMetrics getInstance() {
- return PipeAssignerMetrics.PipeAssignerMetricsHolder.INSTANCE;
- }
-
- private PipeAssignerMetrics() {
- // empty constructor
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
index cb1ba0b9ad9..0e1f0f2b033 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
@@ -114,7 +114,6 @@ public class CombineRequest extends TPipeTransferReq {
version = transferReq.version;
type = transferReq.type;
- body = transferReq.body;
return this;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
index b20904a0e2b..752be403008 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
@@ -99,7 +99,6 @@ public class FetchCombineResultRequest extends TPipeTransferReq {
version = transferReq.version;
type = transferReq.type;
- body = transferReq.body;
return this;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index 00a7e4c46a0..5009cae8a51 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -174,6 +174,14 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
case HANDSHAKE_DATANODE_V1:
{
try {
+ if (PipeConfig.getInstance().isPipeEnableMemoryCheck()
+ && PipeDataNodeResourceManager.memory().getFreeMemorySizeInBytes()
+ < PipeConfig.getInstance().getPipeMinimumReceiverMemory()) {
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(),
+ "The receiver memory is not enough to handle the handshake request from datanode."));
+ }
return handleTransferHandshakeV1(
PipeTransferDataNodeHandshakeV1Req.fromTPipeTransferReq(req));
} finally {
@@ -184,6 +192,14 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
case HANDSHAKE_DATANODE_V2:
{
try {
+ if (PipeConfig.getInstance().isPipeEnableMemoryCheck()
+ && PipeDataNodeResourceManager.memory().getFreeMemorySizeInBytes()
+ < PipeConfig.getInstance().getPipeMinimumReceiverMemory()) {
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(),
+ "The receiver memory is not enough to handle the handshake request from datanode."));
+ }
return handleTransferHandshakeV2(
PipeTransferDataNodeHandshakeV2Req.fromTPipeTransferReq(req));
} finally {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java
index b58d934988f..3afe5eb62e1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java
@@ -21,9 +21,9 @@ package org.apache.iotdb.db.pipe.resource;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.resource.snapshot.PipeSnapshotResourceManager;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +52,9 @@ public class PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner {
LOGGER.info(
"Pipe hardlink dir found, deleting it: {}, result: {}",
pipeHardLinkDir,
- FileUtils.deleteQuietly(pipeHardLinkDir));
+ // For child directories, we need them to recover each pipe's progress
+ // Hence we do not delete them here, only delete the child files (older version)
+ FileUtils.deleteFileChildrenQuietly(pipeHardLinkDir));
}
}
}
@@ -65,7 +67,7 @@ public class PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner {
+ PipeSnapshotResourceManager.PIPE_SNAPSHOT_DIR_NAME);
if (pipeConsensusDir.isDirectory()) {
LOGGER.info("Pipe snapshot dir found, deleting it: {},", pipeConsensusDir);
- org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(pipeConsensusDir);
+ FileUtils.deleteFileOrDirectory(pipeConsensusDir);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
index 573106e45c2..aaf9eff454b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
@@ -19,24 +19,19 @@
package org.apache.iotdb.db.pipe.resource;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.resource.log.PipeLogManager;
import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager;
import org.apache.iotdb.commons.pipe.resource.snapshot.PipeSnapshotResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.pipe.resource.ref.PipeDataNodePhantomReferenceManager;
import org.apache.iotdb.db.pipe.resource.snapshot.PipeDataNodeSnapshotResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeCompactionManager;
import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.hardlink.PipeWALHardlinkResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.selfhost.PipeWALSelfHostResourceManager;
-
-import java.util.concurrent.atomic.AtomicReference;
public class PipeDataNodeResourceManager {
private final PipeTsFileResourceManager pipeTsFileResourceManager;
- private final AtomicReference<PipeWALResourceManager> pipeWALResourceManager;
+ private final PipeCompactionManager pipeCompactionManager;
private final PipeSnapshotResourceManager pipeSnapshotResourceManager;
private final PipeMemoryManager pipeMemoryManager;
private final PipeLogManager pipeLogManager;
@@ -46,18 +41,8 @@ public class PipeDataNodeResourceManager {
return PipeResourceManagerHolder.INSTANCE.pipeTsFileResourceManager;
}
- public static PipeWALResourceManager wal() {
- if (PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get() == null) {
- synchronized (PipeResourceManagerHolder.INSTANCE) {
- if (PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get() == null) {
- PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.set(
- PipeConfig.getInstance().getPipeHardLinkWALEnabled()
- ? new PipeWALHardlinkResourceManager()
- : new PipeWALSelfHostResourceManager());
- }
- }
- }
- return PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get();
+ public static PipeCompactionManager compaction() {
+ return PipeResourceManagerHolder.INSTANCE.pipeCompactionManager;
}
public static PipeSnapshotResourceManager snapshot() {
@@ -80,7 +65,7 @@ public class PipeDataNodeResourceManager {
private PipeDataNodeResourceManager() {
pipeTsFileResourceManager = new PipeTsFileResourceManager();
- pipeWALResourceManager = new AtomicReference<>();
+ pipeCompactionManager = new PipeCompactionManager();
pipeSnapshotResourceManager = new PipeDataNodeSnapshotResourceManager();
pipeMemoryManager = new PipeMemoryManager();
pipeLogManager = new PipeLogManager();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
index 690a7c54083..8a87cf8bad0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
@@ -54,9 +54,6 @@ public class PipeMemoryManager {
private volatile long usedMemorySizeInBytesOfTsFiles;
- private static final double FLOATING_MEMORY_RATIO =
- PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion();
-
// Only non-zero memory blocks will be added to this set.
private final Set<PipeMemoryBlock> allocatedBlocks = new HashSet<>();
@@ -104,18 +101,6 @@ public class PipeMemoryManager {
* getTotalNonFloatingMemorySizeInBytes();
}
- public long getAllocatedMemorySizeInBytesOfWAL() {
- return (long)
- (PipeConfig.getInstance().getPipeDataStructureWalMemoryProportion()
- * getTotalNonFloatingMemorySizeInBytes());
- }
-
- public long getAllocatedMemorySizeInBytesOfBatch() {
- return (long)
- (PipeConfig.getInstance().getPipeDataStructureBatchMemoryProportion()
- * getTotalNonFloatingMemorySizeInBytes());
- }
-
public boolean isEnough4TabletParsing() {
return (double) usedMemorySizeInBytesOfTablets + (double) usedMemorySizeInBytesOfTsFiles
< EXCEED_PROTECT_THRESHOLD * allowedMaxMemorySizeInBytesOfTabletsAndTsFiles()
@@ -648,15 +633,29 @@ public class PipeMemoryManager {
return usedMemorySizeInBytesOfTsFiles;
}
+ public long getAllocatedMemorySizeInBytesOfBatch() {
+ return (long)
+ (PipeConfig.getInstance().getPipeDataStructureBatchMemoryProportion()
+ * getTotalNonFloatingMemorySizeInBytes());
+ }
+
public long getFreeMemorySizeInBytes() {
return TOTAL_MEMORY_SIZE_IN_BYTES - usedMemorySizeInBytes;
}
public static long getTotalNonFloatingMemorySizeInBytes() {
- return (long) (TOTAL_MEMORY_SIZE_IN_BYTES * (1 - FLOATING_MEMORY_RATIO));
+ return (long)
+ (TOTAL_MEMORY_SIZE_IN_BYTES
+ * (1 - PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion()));
}
public static long getTotalFloatingMemorySizeInBytes() {
- return (long) (TOTAL_MEMORY_SIZE_IN_BYTES * FLOATING_MEMORY_RATIO);
+ return (long)
+ (TOTAL_MEMORY_SIZE_IN_BYTES
+ * PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion());
+ }
+
+ public static long getTotalMemorySizeInBytes() {
+ return TOTAL_MEMORY_SIZE_IN_BYTES;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
index cd9a0a79cae..d82c26144d6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
@@ -247,8 +247,8 @@ public class PipeMemoryWeightUtil {
return totalSizeInBytes;
}
- public static long calculateBatchDataRamBytesUsed(BatchData batchData) {
- long totalSizeInBytes = 0;
+ public static int calculateBatchDataRamBytesUsed(BatchData batchData) {
+ int totalSizeInBytes = 0;
// timestamp
totalSizeInBytes += 8;
@@ -263,16 +263,16 @@ public class PipeMemoryWeightUtil {
continue;
}
// consider variable references (plus 8) and memory alignment (round up to 8)
- totalSizeInBytes += roundUpToMultiple(primitiveType.getSize() + 8L, 8);
+ totalSizeInBytes += roundUpToMultiple(primitiveType.getSize() + 8, 8);
}
} else {
if (type.isBinary()) {
final Binary binary = batchData.getBinary();
// refer to org.apache.tsfile.utils.TsPrimitiveType.TsBinary.getSize
totalSizeInBytes +=
- roundUpToMultiple((binary == null ? 8 : binary.ramBytesUsed()) + 8L, 8);
+ roundUpToMultiple((binary == null ? 8 : binary.getLength() + 8) + 8, 8);
} else {
- totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8L, 8);
+ totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8, 8);
}
}
}
@@ -287,7 +287,7 @@ public class PipeMemoryWeightUtil {
* @param n The specified multiple.
* @return The nearest multiple of n greater than or equal to num.
*/
- private static long roundUpToMultiple(long num, int n) {
+ private static int roundUpToMultiple(int num, int n) {
if (n == 0) {
throw new IllegalArgumentException("The multiple n must be greater than 0");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
new file mode 100644
index 00000000000..29d880c720d
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.resource.tsfile;
+
+import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
+import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtaskLifeCycle;
+import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeRealtimePriorityBlockingQueue;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.pipe.api.event.Event;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+public class PipeCompactionManager {
+
+ private final Set<PipeConnectorSubtaskLifeCycle> pipeConnectorSubtaskLifeCycles =
+ new CopyOnWriteArraySet<>();
+
+ public void registerPipeConnectorSubtaskLifeCycle(
+ final PipeConnectorSubtaskLifeCycle pipeConnectorSubtaskLifeCycle) {
+ pipeConnectorSubtaskLifeCycles.add(pipeConnectorSubtaskLifeCycle);
+ }
+
+ public void deregisterPipeConnectorSubtaskLifeCycle(
+ final PipeConnectorSubtaskLifeCycle pipeConnectorSubtaskLifeCycle) {
+ pipeConnectorSubtaskLifeCycles.remove(pipeConnectorSubtaskLifeCycle);
+ }
+
+ public void emitResult(
+ final String storageGroupName,
+ final String dataRegionId,
+ final long timePartition,
+ final List<TsFileResource> seqFileResources,
+ final List<TsFileResource> unseqFileResources,
+ final List<TsFileResource> targetFileResources) {
+ final Set<TsFileResource> sourceFileResources = new HashSet<>(seqFileResources);
+ sourceFileResources.addAll(unseqFileResources);
+
+ for (final PipeConnectorSubtaskLifeCycle lifeCycle : pipeConnectorSubtaskLifeCycles) {
+ final UnboundedBlockingPendingQueue<Event> pendingQueue = lifeCycle.getPendingQueue();
+ // TODO: support non realtime priority blocking queue
+ if (pendingQueue instanceof PipeRealtimePriorityBlockingQueue) {
+ final PipeRealtimePriorityBlockingQueue realtimePriorityBlockingQueue =
+ (PipeRealtimePriorityBlockingQueue) pendingQueue;
+ realtimePriorityBlockingQueue.replace(
+ dataRegionId, sourceFileResources, targetFileResources);
+ }
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileMemResource.java
similarity index 63%
copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileMemResource.java
index bf789b9732d..3673afb92e6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileMemResource.java
@@ -23,8 +23,6 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -36,106 +34,25 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-public class PipeTsFileResource implements AutoCloseable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResource.class);
-
- public static final long TSFILE_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
+public class PipeTsFileMemResource extends PipeTsFileResource {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileMemResource.class);
public static final float MEMORY_SUFFICIENT_THRESHOLD = 0.7f;
-
- private final File hardlinkOrCopiedFile;
- private final boolean isTsFile;
-
- /** this TsFileResource is used to track the {@link TsFileResourceStatus} of original TsFile. * */
- private final TsFileResource tsFileResource;
-
- private volatile long fileSize = -1L;
-
- private final AtomicInteger referenceCount;
- private final AtomicLong lastUnpinToZeroTime;
private PipeMemoryBlock allocatedMemoryBlock;
private Map<IDeviceID, List<String>> deviceMeasurementsMap = null;
private Map<IDeviceID, Boolean> deviceIsAlignedMap = null;
private Map<String, TSDataType> measurementDataTypeMap = null;
- public PipeTsFileResource(
- final File hardlinkOrCopiedFile,
- final boolean isTsFile,
- final TsFileResource tsFileResource) {
- this.hardlinkOrCopiedFile = hardlinkOrCopiedFile;
- this.isTsFile = isTsFile;
- this.tsFileResource = tsFileResource;
-
- referenceCount = new AtomicInteger(1);
- lastUnpinToZeroTime = new AtomicLong(Long.MAX_VALUE);
- }
-
- public File getFile() {
- return hardlinkOrCopiedFile;
- }
-
- public boolean isOriginalTsFileDeleted() {
- return isTsFile && Objects.nonNull(tsFileResource) && tsFileResource.isDeleted();
- }
-
- public long getFileSize() {
- if (fileSize == -1L) {
- synchronized (this) {
- if (fileSize == -1L) {
- fileSize = hardlinkOrCopiedFile.length();
- }
- }
- }
- return fileSize;
- }
-
- public long getTsFileResourceSize() {
- return Objects.nonNull(tsFileResource) ? tsFileResource.calculateRamSize() : 0;
- }
-
- ///////////////////// Reference Count /////////////////////
-
- public int getReferenceCount() {
- return referenceCount.get();
- }
-
- public int increaseAndGetReference() {
- return referenceCount.addAndGet(1);
- }
-
- public int decreaseAndGetReference() {
- final int finalReferenceCount = referenceCount.addAndGet(-1);
- if (finalReferenceCount == 0) {
- lastUnpinToZeroTime.set(System.currentTimeMillis());
- }
- if (finalReferenceCount < 0) {
- LOGGER.warn("PipeTsFileResource's reference count is decreased to below 0.");
- }
- return finalReferenceCount;
- }
-
- public synchronized boolean closeIfOutOfTimeToLive() {
- if (referenceCount.get() <= 0
- && (deviceMeasurementsMap == null // Not cached yet.
- || System.currentTimeMillis() - lastUnpinToZeroTime.get()
- > TSFILE_MIN_TIME_TO_LIVE_IN_MS)) {
- close();
- return true;
- } else {
- return false;
- }
+ public PipeTsFileMemResource() {
+ super(null);
}
@Override
- public synchronized void close() {
+ public void close() {
if (deviceMeasurementsMap != null) {
deviceMeasurementsMap = null;
}
@@ -152,53 +69,39 @@ public class PipeTsFileResource implements AutoCloseable {
allocatedMemoryBlock.close();
allocatedMemoryBlock = null;
}
-
- try {
- Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
- } catch (final Exception e) {
- LOGGER.error(
- "PipeTsFileResource: Failed to delete tsfile {} when closing, because {}. Please MANUALLY delete it.",
- hardlinkOrCopiedFile,
- e.getMessage(),
- e);
- }
-
- LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
}
//////////////////////////// Cache Getter ////////////////////////////
- public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap()
+ public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap(final File tsFile)
throws IOException {
- if (deviceMeasurementsMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ if (deviceMeasurementsMap == null) {
+ cacheObjectsIfAbsent(tsFile);
}
return deviceMeasurementsMap;
}
public synchronized Map<IDeviceID, Boolean> tryGetDeviceIsAlignedMap(
- final boolean cacheOtherMetadata) throws IOException {
- if (deviceIsAlignedMap == null && isTsFile) {
+ final boolean cacheOtherMetadata, final File tsFile) throws IOException {
+ if (deviceIsAlignedMap == null) {
if (cacheOtherMetadata) {
- cacheObjectsIfAbsent();
+ cacheObjectsIfAbsent(tsFile);
} else {
- cacheDeviceIsAlignedMapIfAbsent();
+ cacheDeviceIsAlignedMapIfAbsent(tsFile);
}
}
return deviceIsAlignedMap;
}
- public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap() throws IOException {
- if (measurementDataTypeMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap(final File tsFile)
+ throws IOException {
+ if (measurementDataTypeMap == null) {
+ cacheObjectsIfAbsent(tsFile);
}
return measurementDataTypeMap;
}
- synchronized boolean cacheDeviceIsAlignedMapIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
+ synchronized boolean cacheDeviceIsAlignedMapIfAbsent(final File tsFile) throws IOException {
if (allocatedMemoryBlock != null) {
// This means objects are already cached.
@@ -216,13 +119,13 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"Failed to cacheDeviceIsAlignedMapIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
return false;
}
long memoryRequiredInBytes = 0L;
try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, false)) {
+ new TsFileSequenceReader(tsFile.getPath(), true, false)) {
deviceIsAlignedMap = new HashMap<>();
final TsFileDeviceIterator deviceIsAlignedIterator =
sequenceReader.getAllDevicesIteratorWithIsAligned();
@@ -243,22 +146,16 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
deviceIsAlignedMap = null;
return false;
}
- LOGGER.info(
- "PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.",
- hardlinkOrCopiedFile.getPath());
+ LOGGER.info("PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.", tsFile.getPath());
return true;
}
- synchronized boolean cacheObjectsIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
+ synchronized boolean cacheObjectsIfAbsent(final File tsFile) throws IOException {
if (allocatedMemoryBlock != null) {
if (deviceMeasurementsMap != null) {
return true;
@@ -280,13 +177,13 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"Failed to cacheObjectsIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
return false;
}
long memoryRequiredInBytes = 0L;
try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, true)) {
+ new TsFileSequenceReader(tsFile.getPath(), true, true)) {
deviceMeasurementsMap = sequenceReader.getDeviceMeasurementsMap();
memoryRequiredInBytes +=
PipeMemoryWeightUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap);
@@ -316,15 +213,14 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
deviceIsAlignedMap = null;
deviceMeasurementsMap = null;
measurementDataTypeMap = null;
return false;
}
- LOGGER.info(
- "PipeTsFileResource: Cached objects for tsfile {}.", hardlinkOrCopiedFile.getPath());
+ LOGGER.info("PipeTsFileResource: Cached objects for tsfile {}.", tsFile.getPath());
return true;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
index bf789b9732d..5065821ebe4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
@@ -19,73 +19,33 @@
package org.apache.iotdb.db.pipe.resource.tsfile;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
-
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.read.TsFileDeviceIterator;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
import java.nio.file.Files;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
public class PipeTsFileResource implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResource.class);
- public static final long TSFILE_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
- public static final float MEMORY_SUFFICIENT_THRESHOLD = 0.7f;
-
private final File hardlinkOrCopiedFile;
- private final boolean isTsFile;
-
- /** this TsFileResource is used to track the {@link TsFileResourceStatus} of original TsFile. * */
- private final TsFileResource tsFileResource;
private volatile long fileSize = -1L;
private final AtomicInteger referenceCount;
- private final AtomicLong lastUnpinToZeroTime;
- private PipeMemoryBlock allocatedMemoryBlock;
- private Map<IDeviceID, List<String>> deviceMeasurementsMap = null;
- private Map<IDeviceID, Boolean> deviceIsAlignedMap = null;
- private Map<String, TSDataType> measurementDataTypeMap = null;
- public PipeTsFileResource(
- final File hardlinkOrCopiedFile,
- final boolean isTsFile,
- final TsFileResource tsFileResource) {
+ public PipeTsFileResource(final File hardlinkOrCopiedFile) {
this.hardlinkOrCopiedFile = hardlinkOrCopiedFile;
- this.isTsFile = isTsFile;
- this.tsFileResource = tsFileResource;
referenceCount = new AtomicInteger(1);
- lastUnpinToZeroTime = new AtomicLong(Long.MAX_VALUE);
}
public File getFile() {
return hardlinkOrCopiedFile;
}
- public boolean isOriginalTsFileDeleted() {
- return isTsFile && Objects.nonNull(tsFileResource) && tsFileResource.isDeleted();
- }
-
public long getFileSize() {
if (fileSize == -1L) {
synchronized (this) {
@@ -97,62 +57,30 @@ public class PipeTsFileResource implements AutoCloseable {
return fileSize;
}
- public long getTsFileResourceSize() {
- return Objects.nonNull(tsFileResource) ? tsFileResource.calculateRamSize() : 0;
- }
-
///////////////////// Reference Count /////////////////////
public int getReferenceCount() {
return referenceCount.get();
}
- public int increaseAndGetReference() {
- return referenceCount.addAndGet(1);
+ public void increaseReferenceCount() {
+ referenceCount.addAndGet(1);
}
- public int decreaseAndGetReference() {
+ public boolean decreaseReferenceCount() {
final int finalReferenceCount = referenceCount.addAndGet(-1);
if (finalReferenceCount == 0) {
- lastUnpinToZeroTime.set(System.currentTimeMillis());
+ close();
+ return true;
}
if (finalReferenceCount < 0) {
LOGGER.warn("PipeTsFileResource's reference count is decreased to below 0.");
}
- return finalReferenceCount;
- }
-
- public synchronized boolean closeIfOutOfTimeToLive() {
- if (referenceCount.get() <= 0
- && (deviceMeasurementsMap == null // Not cached yet.
- || System.currentTimeMillis() - lastUnpinToZeroTime.get()
- > TSFILE_MIN_TIME_TO_LIVE_IN_MS)) {
- close();
- return true;
- } else {
- return false;
- }
+ return false;
}
@Override
public synchronized void close() {
- if (deviceMeasurementsMap != null) {
- deviceMeasurementsMap = null;
- }
-
- if (deviceIsAlignedMap != null) {
- deviceIsAlignedMap = null;
- }
-
- if (measurementDataTypeMap != null) {
- measurementDataTypeMap = null;
- }
-
- if (allocatedMemoryBlock != null) {
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
- }
-
try {
Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
} catch (final Exception e) {
@@ -165,166 +93,4 @@ public class PipeTsFileResource implements AutoCloseable {
LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
}
-
- //////////////////////////// Cache Getter ////////////////////////////
-
- public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap()
- throws IOException {
- if (deviceMeasurementsMap == null && isTsFile) {
- cacheObjectsIfAbsent();
- }
- return deviceMeasurementsMap;
- }
-
- public synchronized Map<IDeviceID, Boolean> tryGetDeviceIsAlignedMap(
- final boolean cacheOtherMetadata) throws IOException {
- if (deviceIsAlignedMap == null && isTsFile) {
- if (cacheOtherMetadata) {
- cacheObjectsIfAbsent();
- } else {
- cacheDeviceIsAlignedMapIfAbsent();
- }
- }
- return deviceIsAlignedMap;
- }
-
- public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap() throws IOException {
- if (measurementDataTypeMap == null && isTsFile) {
- cacheObjectsIfAbsent();
- }
- return measurementDataTypeMap;
- }
-
- synchronized boolean cacheDeviceIsAlignedMapIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
- if (allocatedMemoryBlock != null) {
- // This means objects are already cached.
- return true;
- }
-
- // See if pipe memory is sufficient to be allocated for TsFileSequenceReader.
- // Only allocate when pipe memory used is less than 50%, because memory here
- // is hard to shrink and may consume too much memory.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(
- PipeConfig.getInstance().getPipeMemoryAllocateForTsFileSequenceReaderInBytes(),
- MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "Failed to cacheDeviceIsAlignedMapIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- return false;
- }
-
- long memoryRequiredInBytes = 0L;
- try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, false)) {
- deviceIsAlignedMap = new HashMap<>();
- final TsFileDeviceIterator deviceIsAlignedIterator =
- sequenceReader.getAllDevicesIteratorWithIsAligned();
- while (deviceIsAlignedIterator.hasNext()) {
- final Pair<IDeviceID, Boolean> deviceIsAlignedPair = deviceIsAlignedIterator.next();
- deviceIsAlignedMap.put(deviceIsAlignedPair.getLeft(), deviceIsAlignedPair.getRight());
- }
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfIDeviceId2Bool(deviceIsAlignedMap);
- }
- // Release memory of TsFileSequenceReader.
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
-
- // Allocate again for the cached objects.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(memoryRequiredInBytes, MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- deviceIsAlignedMap = null;
- return false;
- }
-
- LOGGER.info(
- "PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.",
- hardlinkOrCopiedFile.getPath());
- return true;
- }
-
- synchronized boolean cacheObjectsIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
- if (allocatedMemoryBlock != null) {
- if (deviceMeasurementsMap != null) {
- return true;
- } else {
- // Recalculate it again because only deviceIsAligned map is cached
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
- }
- }
-
- // See if pipe memory is sufficient to be allocated for TsFileSequenceReader.
- // Only allocate when pipe memory used is less than 50%, because memory here
- // is hard to shrink and may consume too much memory.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(
- PipeConfig.getInstance().getPipeMemoryAllocateForTsFileSequenceReaderInBytes(),
- MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "Failed to cacheObjectsIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- return false;
- }
-
- long memoryRequiredInBytes = 0L;
- try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, true)) {
- deviceMeasurementsMap = sequenceReader.getDeviceMeasurementsMap();
- memoryRequiredInBytes +=
- PipeMemoryWeightUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap);
-
- if (Objects.isNull(deviceIsAlignedMap)) {
- deviceIsAlignedMap = new HashMap<>();
- final TsFileDeviceIterator deviceIsAlignedIterator =
- sequenceReader.getAllDevicesIteratorWithIsAligned();
- while (deviceIsAlignedIterator.hasNext()) {
- final Pair<IDeviceID, Boolean> deviceIsAlignedPair = deviceIsAlignedIterator.next();
- deviceIsAlignedMap.put(deviceIsAlignedPair.getLeft(), deviceIsAlignedPair.getRight());
- }
- }
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfIDeviceId2Bool(deviceIsAlignedMap);
-
- measurementDataTypeMap = sequenceReader.getFullPathDataTypeMap();
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfStr2TSDataType(measurementDataTypeMap);
- }
- // Release memory of TsFileSequenceReader.
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
-
- // Allocate again for the cached objects.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(memoryRequiredInBytes, MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- deviceIsAlignedMap = null;
- deviceMeasurementsMap = null;
- measurementDataTypeMap = null;
- return false;
- }
-
- LOGGER.info(
- "PipeTsFileResource: Cached objects for tsfile {}.", hardlinkOrCopiedFile.getPath());
- return true;
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
index d00318ae561..37b3014a499 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
@@ -22,8 +22,7 @@ package org.apache.iotdb.db.pipe.resource.tsfile;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -32,88 +31,30 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
import java.io.File;
import java.io.IOException;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
+import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
public class PipeTsFileResourceManager {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResourceManager.class);
- private final Map<String, PipeTsFileResource> hardlinkOrCopiedFileToPipeTsFileResourceMap =
+ // This is used to hold the assigner pinned tsFiles.
+ // Also, it is used to provide metadata cache of the tsFile, and is shared by all the pipe's
+ // tsFiles.
+ private final Map<String, PipeTsFileMemResource> hardlinkOrCopiedFileToTsFileMemResourceMap =
new ConcurrentHashMap<>();
- private final PipeTsFileResourceSegmentLock segmentLock = new PipeTsFileResourceSegmentLock();
-
- public PipeTsFileResourceManager() {
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "PipeTsFileResourceManager#ttlCheck()",
- this::tryTtlCheck,
- Math.max(PipeTsFileResource.TSFILE_MIN_TIME_TO_LIVE_IN_MS / 1000, 1));
- }
- private void tryTtlCheck() {
- try {
- ttlCheck();
- } catch (final InterruptedException e) {
- Thread.currentThread().interrupt();
- LOGGER.warn("failed to try lock when checking TTL because of interruption", e);
- } catch (final Exception e) {
- LOGGER.warn("failed to check TTL of PipeTsFileResource: ", e);
- }
- }
-
- private void ttlCheck() throws InterruptedException {
- final Iterator<Map.Entry<String, PipeTsFileResource>> iterator =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.entrySet().iterator();
- final long timeout =
- PipeConfig.getInstance().getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds() >> 1;
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeTsFileResourceManager.class,
- PipeConfig.getInstance().getPipeTsFilePinMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeTsFilePinMaxLogIntervalRounds(),
- hardlinkOrCopiedFileToPipeTsFileResourceMap.size());
- final StringBuilder logBuilder = new StringBuilder();
- while (iterator.hasNext()) {
- final Map.Entry<String, PipeTsFileResource> entry = iterator.next();
-
- final String hardlinkOrCopiedFile = entry.getKey();
- if (!segmentLock.tryLock(new File(hardlinkOrCopiedFile), timeout, TimeUnit.SECONDS)) {
- LOGGER.warn(
- "failed to try lock when checking TTL for file {} because of timeout ({}s)",
- hardlinkOrCopiedFile,
- timeout);
- continue;
- }
-
- try {
- if (entry.getValue().closeIfOutOfTimeToLive()) {
- iterator.remove();
- } else {
- logBuilder.append(
- String.format(
- "<%s , %d times, %d bytes> ",
- entry.getKey(),
- entry.getValue().getReferenceCount(),
- entry.getValue().getFileSize()));
- }
- } catch (final Exception e) {
- LOGGER.warn("failed to close PipeTsFileResource when checking TTL: ", e);
- } finally {
- segmentLock.unlock(new File(hardlinkOrCopiedFile));
- }
- }
- if (logBuilder.length() > 0) {
- logger.ifPresent(l -> l.info("Pipe file {}are still referenced", logBuilder));
- }
- }
+ // PipeName -> TsFilePath -> PipeTsFileResource
+ private final Map<String, Map<String, PipeTsFileResource>>
+ hardlinkOrCopiedFileToPipeTsFileResourceMap = new ConcurrentHashMap<>();
+ private final PipeTsFileResourceSegmentLock segmentLock = new PipeTsFileResourceSegmentLock();
/**
* Given a file, create a hardlink or copy it to pipe dir, maintain a reference count for the
@@ -131,19 +72,16 @@ public class PipeTsFileResourceManager {
* @param file tsfile, resource file or mod file. can be original file or hardlink/copy of
* original file
* @param isTsFile {@code true} to create hardlink, {@code false} to copy file
- * @param tsFileResource the TsFileResource of original TsFile. Ignored if {@param isTsFile} is
- * {@code false}.
* @return the hardlink or copied file
* @throws IOException when create hardlink or copy file failed
*/
public File increaseFileReference(
- final File file, final boolean isTsFile, final TsFileResource tsFileResource)
- throws IOException {
+ final File file, final boolean isTsFile, final @Nonnull String pipeName) throws IOException {
// If the file is already a hardlink or copied file,
// just increase reference count and return it
segmentLock.lock(file);
try {
- if (increaseReferenceIfExists(file)) {
+ if (increaseReferenceIfExists(file, isTsFile, pipeName)) {
return file;
}
} finally {
@@ -152,11 +90,12 @@ public class PipeTsFileResourceManager {
// If the file is not a hardlink or copied file, check if there is a related hardlink or
// copied file in pipe dir. if so, increase reference count and return it
- final File hardlinkOrCopiedFile = getHardlinkOrCopiedFileInPipeDir(file);
+ final File hardlinkOrCopiedFile = getHardlinkOrCopiedFileInPipeDir(file, pipeName);
segmentLock.lock(hardlinkOrCopiedFile);
try {
- if (increaseReferenceIfExists(hardlinkOrCopiedFile)) {
+ if (increaseReferenceIfExists(hardlinkOrCopiedFile, isTsFile, pipeName)) {
return hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
.get(hardlinkOrCopiedFile.getPath())
.getFile();
}
@@ -171,27 +110,55 @@ public class PipeTsFileResourceManager {
// If the file is not a hardlink or copied file, and there is no related hardlink or copied
// file in pipe dir, create a hardlink or copy it to pipe dir, maintain a reference count for
// the hardlink or copied file, and return the hardlink or copied file.
- hardlinkOrCopiedFileToPipeTsFileResourceMap.put(
- resultFile.getPath(), new PipeTsFileResource(resultFile, isTsFile, tsFileResource));
+ hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .put(resultFile.getPath(), new PipeTsFileResource(resultFile));
+
+ increaseMemReference(resultFile, isTsFile);
+
return resultFile;
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
}
}
- private boolean increaseReferenceIfExists(final File file) {
+ private boolean increaseReferenceIfExists(
+ final File file, final boolean isTsFile, final @Nonnull String pipeName) {
+ final String path = file.getPath();
final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(file.getPath());
+ hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .get(path);
if (resource != null) {
- resource.increaseAndGetReference();
+ resource.increaseReferenceCount();
+ increaseMemReference(file, isTsFile);
return true;
}
return false;
}
- public static File getHardlinkOrCopiedFileInPipeDir(final File file) throws IOException {
+ private void increaseMemReference(final File file, final boolean isTsFile) {
+ if (!isTsFile) {
+ return;
+ }
+ // Increase the assigner's file to avoid hard-link or memory cache cleaning
+ // Note that it does not exist for historical files
+ hardlinkOrCopiedFileToTsFileMemResourceMap.compute(
+ getCommonFilePath(file),
+ (k, v) -> {
+ if (Objects.isNull(v)) {
+ return new PipeTsFileMemResource();
+ } else {
+ v.increaseReferenceCount();
+ return v;
+ }
+ });
+ }
+
+ public static File getHardlinkOrCopiedFileInPipeDir(final File file, final String pipeName)
+ throws IOException {
try {
- return new File(getPipeTsFileDirPath(file), getRelativeFilePath(file));
+ return new File(getPipeTsFileDirPath(file, pipeName), getRelativeFilePath(file));
} catch (final Exception e) {
throw new IOException(
String.format(
@@ -202,22 +169,27 @@ public class PipeTsFileResourceManager {
}
}
- private static String getPipeTsFileDirPath(File file) throws IOException {
+ private static String getPipeTsFileDirPath(File file, final String pipeName) throws IOException {
while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
- && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)) {
+ && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
+ && !file.getName().equals(PipeConfig.getInstance().getPipeHardlinkBaseDirName())) {
file = file.getParentFile();
}
return file.getParentFile().getCanonicalPath()
+ File.separator
+ PipeConfig.getInstance().getPipeHardlinkBaseDirName()
+ File.separator
- + PipeConfig.getInstance().getPipeHardlinkTsFileDirName();
+ + PipeConfig.getInstance().getPipeHardlinkTsFileDirName()
+ + (Objects.nonNull(pipeName) ? File.separator + pipeName : "");
}
private static String getRelativeFilePath(File file) {
StringBuilder builder = new StringBuilder(file.getName());
while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
- && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)) {
+ && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
+ && !file.getParentFile()
+ .getName()
+ .equals(PipeConfig.getInstance().getPipeHardlinkTsFileDirName())) {
file = file.getParentFile();
builder =
new StringBuilder(file.getName())
@@ -231,32 +203,70 @@ public class PipeTsFileResourceManager {
* Given a hardlink or copied file, decrease its reference count, if the reference count is 0,
* delete the file. if the given file is not a hardlink or copied file, do nothing.
*
- * @param hardlinkOrCopiedFile the copied or hardlinked file
+ * @param hardlinkOrCopiedFile the copied or hard-linked file
*/
- public void decreaseFileReference(final File hardlinkOrCopiedFile) {
+ public void decreaseFileReference(
+ final File hardlinkOrCopiedFile, final @Nonnull String pipeName) {
segmentLock.lock(hardlinkOrCopiedFile);
try {
final String filePath = hardlinkOrCopiedFile.getPath();
- final PipeTsFileResource resource = hardlinkOrCopiedFileToPipeTsFileResourceMap.get(filePath);
- if (resource != null) {
- resource.decreaseAndGetReference();
+ final PipeTsFileResource resource =
+ hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .get(filePath);
+ if (resource != null && resource.decreaseReferenceCount()) {
+ hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .remove(filePath);
}
+ // Decrease the assigner's file to clear hard-link and memory cache
+ // Note that it does not exist for historical files
+ decreaseMemReferenceIfExists(hardlinkOrCopiedFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
}
}
+ private void decreaseMemReferenceIfExists(final File file) {
+ // Increase the assigner's file to avoid hard-link or memory cache cleaning
+ // Note that it does not exist for historical files
+ final String commonFilePath = getCommonFilePath(file);
+ if (hardlinkOrCopiedFileToTsFileMemResourceMap.containsKey(commonFilePath)
+ && hardlinkOrCopiedFileToTsFileMemResourceMap
+ .get(commonFilePath)
+ .decreaseReferenceCount()) {
+ hardlinkOrCopiedFileToPipeTsFileResourceMap.remove(commonFilePath);
+ }
+ }
+
+ // Warning: Shall not be called by the assigner
+ private String getCommonFilePath(final @Nonnull File file) {
+ // If the parent or grandparent is null then this is testing scenario
+ // Skip the "pipeName" of this file
+ return Objects.isNull(file.getParentFile())
+ || Objects.isNull(file.getParentFile().getParentFile())
+ ? file.getPath()
+ : file.getParentFile().getParent() + File.separator + file.getName();
+ }
+
/**
* Get the reference count of the file.
*
* @param hardlinkOrCopiedFile the copied or hardlinked file
* @return the reference count of the file
*/
- public int getFileReferenceCount(final File hardlinkOrCopiedFile) {
+ @TestOnly
+ public int getFileReferenceCount(
+ final File hardlinkOrCopiedFile, final @Nullable String pipeName) {
segmentLock.lock(hardlinkOrCopiedFile);
try {
- final String filePath = hardlinkOrCopiedFile.getPath();
- final PipeTsFileResource resource = hardlinkOrCopiedFileToPipeTsFileResourceMap.get(filePath);
+ final PipeTsFileResource resource =
+ Objects.nonNull(pipeName)
+ ? hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .get(hardlinkOrCopiedFile.getPath())
+ : hardlinkOrCopiedFileToTsFileMemResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedFile));
return resource != null ? resource.getReferenceCount() : 0;
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
@@ -272,9 +282,13 @@ public class PipeTsFileResourceManager {
public boolean cacheObjectsIfAbsent(final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource != null && resource.cacheObjectsIfAbsent();
+ if (hardlinkOrCopiedTsFile.getParentFile() == null
+ || hardlinkOrCopiedTsFile.getParentFile().getParentFile() == null) {
+ return false;
+ }
+ final PipeTsFileMemResource resource =
+ hardlinkOrCopiedFileToTsFileMemResourceMap.get(getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource != null && resource.cacheObjectsIfAbsent(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -284,9 +298,9 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetDeviceMeasurementsMap();
+ final PipeTsFileMemResource resource =
+ hardlinkOrCopiedFileToTsFileMemResourceMap.get(getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null ? null : resource.tryGetDeviceMeasurementsMap(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -296,9 +310,11 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile, final boolean cacheOtherMetadata) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetDeviceIsAlignedMap(cacheOtherMetadata);
+ final PipeTsFileMemResource resource =
+ hardlinkOrCopiedFileToTsFileMemResourceMap.get(getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null
+ ? null
+ : resource.tryGetDeviceIsAlignedMap(cacheOtherMetadata, hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -308,38 +324,47 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetMeasurementDataTypeMap();
+ final PipeTsFileMemResource resource =
+ hardlinkOrCopiedFileToTsFileMemResourceMap.get(getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null
+ ? null
+ : resource.tryGetMeasurementDataTypeMap(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
}
- public void pinTsFileResource(final TsFileResource resource, final boolean withMods)
+ public void pinTsFileResource(
+ final TsFileResource resource, final boolean withMods, final String pipeName)
throws IOException {
- increaseFileReference(resource.getTsFile(), true, resource);
+ increaseFileReference(resource.getTsFile(), true, pipeName);
if (withMods && resource.getModFile().exists()) {
- increaseFileReference(new File(resource.getModFile().getFilePath()), false, null);
+ increaseFileReference(new File(resource.getModFile().getFilePath()), false, pipeName);
}
}
- public void unpinTsFileResource(final TsFileResource resource) throws IOException {
- final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile());
- decreaseFileReference(pinnedFile);
+ public void unpinTsFileResource(final TsFileResource resource, final String pipeName)
+ throws IOException {
+ final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile(), pipeName);
+ decreaseFileReference(pinnedFile, pipeName);
final File modFile = new File(pinnedFile + ModificationFile.FILE_SUFFIX);
if (modFile.exists()) {
- decreaseFileReference(modFile);
+ decreaseFileReference(modFile, pipeName);
}
}
- public int getLinkedTsfileCount() {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.size();
+ public int getLinkedTsFileCount(final @Nonnull String pipeName) {
+ return hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .size();
}
- public long getTotalLinkedTsfileSize() {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.values().stream()
+ public long getTotalLinkedTsFileSize(final @Nonnull String pipeName) {
+ return hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .values()
+ .stream()
.mapToLong(
resource -> {
try {
@@ -351,47 +376,4 @@ public class PipeTsFileResourceManager {
})
.sum();
}
-
- /**
- * Get the total size of linked TsFiles whose original TsFile is deleted (by compaction or else)
- */
- public long getTotalLinkedButDeletedTsfileSize() {
- try {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.values().parallelStream()
- .filter(PipeTsFileResource::isOriginalTsFileDeleted)
- .mapToLong(
- resource -> {
- try {
- return resource.getFileSize();
- } catch (Exception e) {
- LOGGER.warn(
- "failed to get file size of linked but deleted TsFile {}: ", resource, e);
- return 0;
- }
- })
- .sum();
- } catch (final Exception e) {
- LOGGER.warn("failed to get total size of linked but deleted TsFiles: ", e);
- return 0;
- }
- }
-
- public long getTotalLinkedButDeletedTsFileResourceRamSize() {
- long totalLinkedButDeletedTsfileResourceRamSize = 0;
- try {
- for (final Map.Entry<String, PipeTsFileResource> resourceEntry :
- hardlinkOrCopiedFileToPipeTsFileResourceMap.entrySet()) {
- final PipeTsFileResource pipeTsFileResource = resourceEntry.getValue();
- // If the original TsFile is not deleted, the memory of the resource is not counted
- // because the memory of the resource is controlled by TsFileResourceManager.
- if (pipeTsFileResource.isOriginalTsFileDeleted()) {
- totalLinkedButDeletedTsfileResourceRamSize += pipeTsFileResource.getTsFileResourceSize();
- }
- }
- return totalLinkedButDeletedTsfileResourceRamSize;
- } catch (final Exception e) {
- LOGGER.warn("failed to get total size of linked but deleted TsFiles resource ram size: ", e);
- return totalLinkedButDeletedTsfileResourceRamSize;
- }
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
deleted file mode 100644
index 9d1e530a19d..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal;
-
-import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
-import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-public abstract class PipeWALResource implements Closeable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALResource.class);
-
- protected final WALEntryHandler walEntryHandler;
-
- private final AtomicInteger referenceCount;
-
- public static final long WAL_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
- private final AtomicLong lastLogicalPinTime;
- private final AtomicBoolean isPhysicallyPinned;
-
- protected PipeWALResource(WALEntryHandler walEntryHandler) {
- this.walEntryHandler = walEntryHandler;
-
- referenceCount = new AtomicInteger(0);
-
- lastLogicalPinTime = new AtomicLong(0);
- isPhysicallyPinned = new AtomicBoolean(false);
- }
-
- public final void pin() throws PipeRuntimeNonCriticalException {
- if (referenceCount.get() == 0) {
- if (!isPhysicallyPinned.get()) {
- try {
- pinInternal();
- } catch (MemTablePinException e) {
- throw new PipeRuntimeNonCriticalException(
- String.format(
- "failed to pin wal %d, because %s",
- walEntryHandler.getMemTableId(), e.getMessage()));
- }
- isPhysicallyPinned.set(true);
- LOGGER.info("wal {} is pinned by pipe engine", walEntryHandler.getMemTableId());
- } // else means the wal is already pinned, do nothing
-
- // no matter the wal is pinned or not, update the last pin time
- lastLogicalPinTime.set(System.currentTimeMillis());
- }
-
- referenceCount.incrementAndGet();
- }
-
- protected abstract void pinInternal()
- throws MemTablePinException, PipeRuntimeNonCriticalException;
-
- public final void unpin() throws PipeRuntimeNonCriticalException {
- final int finalReferenceCount = referenceCount.get();
-
- if (finalReferenceCount == 1) {
- unpinPhysicallyIfOutOfTimeToLive();
- } else if (finalReferenceCount < 1) {
- throw new PipeRuntimeCriticalException(
- String.format(
- "wal %d is unpinned more than pinned, this should not happen",
- walEntryHandler.getMemTableId()));
- }
-
- referenceCount.decrementAndGet();
- }
-
- protected abstract void unpinInternal()
- throws MemTablePinException, PipeRuntimeNonCriticalException;
-
- /**
- * Invalidate the wal if it is unpinned and out of time to live.
- *
- * @return true if the wal is invalidated, false otherwise
- */
- public final boolean invalidateIfPossible() {
- if (referenceCount.get() > 0) {
- return false;
- }
-
- // referenceCount.get() == 0
- return unpinPhysicallyIfOutOfTimeToLive();
- }
-
- /**
- * Unpin the wal if it is out of time to live.
- *
- * @return true if the wal is unpinned physically (then it can be invalidated), false otherwise
- * @throws PipeRuntimeNonCriticalException if failed to unpin WAL of memtable.
- */
- private boolean unpinPhysicallyIfOutOfTimeToLive() {
- if (isPhysicallyPinned.get()) {
- if (System.currentTimeMillis() - lastLogicalPinTime.get() > WAL_MIN_TIME_TO_LIVE_IN_MS) {
- try {
- unpinInternal();
- } catch (MemTablePinException e) {
- throw new PipeRuntimeNonCriticalException(
- String.format(
- "failed to unpin wal %d, because %s",
- walEntryHandler.getMemTableId(), e.getMessage()));
- }
- isPhysicallyPinned.set(false);
- LOGGER.info(
- "wal {} is unpinned by pipe engine when checking time to live",
- walEntryHandler.getMemTableId());
- return true;
- } else {
- return false;
- }
- } else {
- LOGGER.info(
- "wal {} is not pinned physically when checking time to live",
- walEntryHandler.getMemTableId());
- return true;
- }
- }
-
- @Override
- public final void close() {
- if (isPhysicallyPinned.get()) {
- try {
- unpinInternal();
- } catch (MemTablePinException e) {
- LOGGER.error(
- "failed to unpin wal {} when closing pipe wal resource, because {}",
- walEntryHandler.getMemTableId(),
- e.getMessage());
- }
- isPhysicallyPinned.set(false);
- LOGGER.info(
- "wal {} is unpinned by pipe engine when closing pipe wal resource",
- walEntryHandler.getMemTableId());
- }
-
- referenceCount.set(0);
- }
-
- public int getReferenceCount() {
- return referenceCount.get();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
deleted file mode 100644
index 9c51d79daad..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ConcurrentModificationException;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantLock;
-
-public abstract class PipeWALResourceManager {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALResourceManager.class);
-
- protected final Map<Long, PipeWALResource> memtableIdToPipeWALResourceMap;
-
- private static final int SEGMENT_LOCK_COUNT = 32;
- private final ReentrantLock[] memTableIdSegmentLocks;
-
- protected PipeWALResourceManager() {
- // memTableIdToPipeWALResourceMap can be concurrently accessed by multiple threads
- memtableIdToPipeWALResourceMap = new ConcurrentHashMap<>();
-
- memTableIdSegmentLocks = new ReentrantLock[SEGMENT_LOCK_COUNT];
- for (int i = 0; i < SEGMENT_LOCK_COUNT; i++) {
- memTableIdSegmentLocks[i] = new ReentrantLock();
- }
-
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "PipeWALResourceManager#ttlCheck()",
- this::ttlCheck,
- Math.max(PipeWALResource.WAL_MIN_TIME_TO_LIVE_IN_MS / 1000, 1));
- }
-
- @SuppressWarnings("java:S2222")
- private void ttlCheck() {
- final Iterator<Map.Entry<Long, PipeWALResource>> iterator =
- memtableIdToPipeWALResourceMap.entrySet().iterator();
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeWALResourceManager.class,
- PipeConfig.getInstance().getPipeWalPinMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeWalPinMaxLogIntervalRounds(),
- memtableIdToPipeWALResourceMap.size());
-
- final StringBuilder logBuilder = new StringBuilder();
- try {
- while (iterator.hasNext()) {
- final Map.Entry<Long, PipeWALResource> entry = iterator.next();
- final ReentrantLock lock =
- memTableIdSegmentLocks[(int) (entry.getKey() % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- if (entry.getValue().invalidateIfPossible()) {
- iterator.remove();
- } else {
- logBuilder.append(
- String.format(
- "<%d , %d times> ", entry.getKey(), entry.getValue().getReferenceCount()));
- }
- } finally {
- lock.unlock();
- }
- }
- } catch (final ConcurrentModificationException e) {
- LOGGER.error(
- "Concurrent modification issues happened, skipping the WAL in this round of ttl check",
- e);
- } finally {
- if (logBuilder.length() > 0) {
- logger.ifPresent(l -> l.info("WAL {}are still referenced", logBuilder));
- }
- }
- }
-
- public final void pin(final WALEntryHandler walEntryHandler) throws IOException {
- final long memTableId = walEntryHandler.getMemTableId();
- final ReentrantLock lock = memTableIdSegmentLocks[(int) (memTableId % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- pinInternal(memTableId, walEntryHandler);
- } finally {
- lock.unlock();
- }
- }
-
- protected abstract void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler)
- throws IOException;
-
- public final void unpin(final WALEntryHandler walEntryHandler) throws IOException {
- final long memTableId = walEntryHandler.getMemTableId();
- final ReentrantLock lock = memTableIdSegmentLocks[(int) (memTableId % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- unpinInternal(memTableId, walEntryHandler);
- } finally {
- lock.unlock();
- }
- }
-
- protected abstract void unpinInternal(
- final long memTableId, final WALEntryHandler walEntryHandler) throws IOException;
-
- public int getPinnedWalCount() {
- return Objects.nonNull(memtableIdToPipeWALResourceMap)
- ? memtableIdToPipeWALResourceMap.size()
- : 0;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
deleted file mode 100644
index f1ad513ccc4..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.hardlink;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALHardlinkResource extends PipeWALResource {
-
- private final PipeWALHardlinkResourceManager resourceManager;
-
- protected PipeWALHardlinkResource(
- WALEntryHandler walEntryHandler, PipeWALHardlinkResourceManager resourceManager) {
- super(walEntryHandler);
- this.resourceManager = resourceManager;
- }
-
- @Override
- protected void pinInternal() throws MemTablePinException {
- // TODO: hardlink
- walEntryHandler.pinMemTable();
- }
-
- @Override
- protected void unpinInternal() throws MemTablePinException {
- // TODO: hardlink
- walEntryHandler.unpinMemTable();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
deleted file mode 100644
index eebf766dc36..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.hardlink;
-
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.HashMap;
-import java.util.Map;
-
-public class PipeWALHardlinkResourceManager extends PipeWALResourceManager {
-
- @Override
- protected void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap
- .computeIfAbsent(memTableId, id -> new PipeWALHardlinkResource(walEntryHandler, this))
- .pin();
- }
-
- @Override
- protected void unpinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap.get(memTableId).unpin();
- }
-
- //////////////////////////// hardlink related ////////////////////////////
-
- private final Map<String, Integer> hardlinkToReferenceMap = new HashMap<>();
-
- /**
- * given a file, create a hardlink, maintain a reference count for the hardlink, and return the
- * hardlink.
- *
- * <p>if the given file is already a hardlink, increase its reference count and return it.
- *
- * <p>if the given file is a wal, create a hardlink in pipe dir, increase the reference count of
- * the hardlink and return it.
- *
- * @param file wal file. can be original file or the hardlink of original file
- * @return the hardlink
- * @throws IOException when create hardlink failed
- */
- public synchronized File increaseFileReference(final File file) throws IOException {
- // if the file is already a hardlink, just increase reference count and return it
- if (increaseReferenceIfExists(file.getPath())) {
- return file;
- }
-
- // if the file is not a hardlink, check if there is a related hardlink in pipe dir. if so,
- // increase reference count and return it.
- final File hardlink = getHardlinkInPipeWALDir(file);
- if (increaseReferenceIfExists(hardlink.getPath())) {
- return hardlink;
- }
-
- // if the file is a wal, and there is no related hardlink in pipe dir, create a hardlink to pipe
- // dir, maintain a reference count for the hardlink, and return the hardlink.
- hardlinkToReferenceMap.put(hardlink.getPath(), 1);
- return FileUtils.createHardLink(file, hardlink);
- }
-
- private boolean increaseReferenceIfExists(final String path) {
- hardlinkToReferenceMap.computeIfPresent(path, (k, v) -> v + 1);
- return hardlinkToReferenceMap.containsKey(path);
- }
-
- // TODO: Check me! Make sure the file is not a hardlink.
- // TODO: IF user specify a wal by config, will the method work?
- private static File getHardlinkInPipeWALDir(final File file) throws IOException {
- try {
- return new File(getPipeWALDirPath(file), getRelativeFilePath(file));
- } catch (final Exception e) {
- throw new IOException(
- String.format(
- "failed to get hardlink in pipe dir " + "for file %s, it is not a wal",
- file.getPath()),
- e);
- }
- }
-
- private static String getPipeWALDirPath(File file) throws IOException {
- while (!file.getName().equals(IoTDBConstant.WAL_FOLDER_NAME)) {
- file = file.getParentFile();
- }
-
- return file.getParentFile().getCanonicalPath()
- + File.separator
- + IoTDBConstant.DATA_FOLDER_NAME
- + File.separator
- + PipeConfig.getInstance().getPipeHardlinkBaseDirName()
- + File.separator
- + PipeConfig.getInstance().getPipeHardlinkWALDirName();
- }
-
- private static String getRelativeFilePath(File file) {
- StringBuilder builder = new StringBuilder(file.getName());
- while (!file.getParentFile().getName().equals(IoTDBConstant.WAL_FOLDER_NAME)) {
- file = file.getParentFile();
- builder =
- new StringBuilder(file.getName())
- .append(IoTDBConstant.FILE_NAME_SEPARATOR)
- .append(builder);
- }
- return builder.toString();
- }
-
- /**
- * given a hardlink, decrease its reference count, if the reference count is 0, delete the file.
- * if the given file is not a hardlink, do nothing.
- *
- * @param hardlink the hardlinked file
- * @throws IOException when delete file failed
- */
- public synchronized void decreaseFileReference(final File hardlink) throws IOException {
- final Integer updatedReference =
- hardlinkToReferenceMap.computeIfPresent(
- hardlink.getPath(), (file, reference) -> reference - 1);
-
- if (updatedReference != null && updatedReference == 0) {
- Files.deleteIfExists(hardlink.toPath());
- hardlinkToReferenceMap.remove(hardlink.getPath());
- }
- }
-
- @TestOnly
- public synchronized int getFileReferenceCount(final File hardlink) {
- return hardlinkToReferenceMap.getOrDefault(hardlink.getPath(), 0);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
deleted file mode 100644
index e8e03e64a16..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.selfhost;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALSelfHostResource extends PipeWALResource {
-
- public PipeWALSelfHostResource(WALEntryHandler walEntryHandler) {
- super(walEntryHandler);
- }
-
- @Override
- protected void pinInternal() throws MemTablePinException {
- walEntryHandler.pinMemTable();
- }
-
- @Override
- protected void unpinInternal() throws MemTablePinException {
- walEntryHandler.unpinMemTable();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
deleted file mode 100644
index c7fe0accda2..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.selfhost;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALSelfHostResourceManager extends PipeWALResourceManager {
-
- @Override
- protected void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap
- .computeIfAbsent(memTableId, id -> new PipeWALSelfHostResource(walEntryHandler))
- .pin();
- }
-
- @Override
- protected void unpinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap.get(memTableId).unpin();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 9eb5446f0cb..0be487aa7fd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -40,6 +40,9 @@ import org.apache.iotdb.common.rpc.thrift.TTestConnectionResult;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.request.AsyncRequestContext;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.ConfigurationFileUtils;
@@ -272,9 +275,17 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -316,6 +327,18 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig();
+ private final ExecutorService schemaExecutor =
+ new WrappedThreadPoolExecutor(
+ 0,
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount(),
+ 0L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount()),
+ new IoTThreadFactory(ThreadName.SCHEMA_PARALLEL_POOL.getName()),
+ ThreadName.SCHEMA_PARALLEL_POOL.getName(),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+
private static final String SYSTEM = "system";
public DataNodeInternalRPCServiceImpl() {
@@ -1071,6 +1094,10 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
.map(PipeMeta::deserialize4TaskAgent)
.collect(Collectors.toList()));
+ if (Objects.isNull(exceptionMessages)) {
+ return new TPushPipeMetaResp()
+ .setStatus(new TSStatus(TSStatusCode.PIPE_PUSH_META_TIMEOUT.getStatusCode()));
+ }
return exceptionMessages.isEmpty()
? new TPushPipeMetaResp()
.setStatus(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))
@@ -1347,16 +1374,31 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
final List<TSStatus> statusList = Collections.synchronizedList(new ArrayList<>());
final AtomicBoolean hasFailure = new AtomicBoolean(false);
- consensusGroupIdList.parallelStream()
- .forEach(
- consensusGroupId -> {
- final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
- && status.getCode() != TSStatusCode.ONLY_LOGICAL_VIEW.getStatusCode()) {
- hasFailure.set(true);
- }
- statusList.add(status);
- });
+ final Set<Future<?>> schemaFuture = new HashSet<>();
+
+ consensusGroupIdList.forEach(
+ consensusGroupId ->
+ schemaFuture.add(
+ schemaExecutor.submit(
+ () -> {
+ final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && status.getCode() != TSStatusCode.ONLY_LOGICAL_VIEW.getStatusCode()) {
+ hasFailure.set(true);
+ }
+ statusList.add(status);
+ })));
+
+ for (final Future<?> future : schemaFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing internal schema task: ", e);
+ statusList.add(
+ new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ .setMessage(e.toString()));
+ }
+ }
if (hasFailure.get()) {
return RpcUtils.getStatus(statusList);
@@ -1375,15 +1417,30 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
final List<TSStatus> statusList = Collections.synchronizedList(new ArrayList<>());
final AtomicBoolean hasFailure = new AtomicBoolean(false);
- consensusGroupIdList.parallelStream()
- .forEach(
- consensusGroupId -> {
- final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- hasFailure.set(true);
- }
- statusList.add(status);
- });
+ final Set<Future<?>> schemaFuture = new HashSet<>();
+
+ consensusGroupIdList.forEach(
+ consensusGroupId ->
+ schemaFuture.add(
+ schemaExecutor.submit(
+ () -> {
+ final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ hasFailure.set(true);
+ }
+ statusList.add(status);
+ })));
+
+ for (final Future<?> future : schemaFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing internal schema task: ", e);
+ statusList.add(
+ new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ .setMessage(e.toString()));
+ }
+ }
if (hasFailure.get()) {
return RpcUtils.getStatus(statusList);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java
index 687a438f9b3..16ee005a089 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeMetrics;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.queryengine.common.header.ColumnHeader;
import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
@@ -97,7 +97,7 @@ public class ShowPipeTask implements IConfigTask {
if (remainingEventCount == -1 && remainingTime == -1) {
final Pair<Long, Double> remainingEventAndTime =
- PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
+ PipeDataNodeSinglePipeMetrics.getInstance()
.getRemainingEventAndTime(tPipeInfo.getId(), tPipeInfo.getCreationTime());
remainingEventCount = remainingEventAndTime.getLeft();
remainingTime = remainingEventAndTime.getRight();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
index 13b87f5bb83..3c4cc1b7f3d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
@@ -25,10 +25,14 @@ import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.pipe.agent.runtime.PipePeriodicalJobExecutor;
+import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeRemainingEventAndTimeOperator;
+import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
@@ -42,6 +46,8 @@ import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Map;
+
public class DataNodeShutdownHook extends Thread {
private static final Logger logger = LoggerFactory.getLogger(DataNodeShutdownHook.class);
@@ -86,6 +92,27 @@ public class DataNodeShutdownHook extends Thread {
triggerSnapshotForAllDataRegion();
}
+ long startTime = System.currentTimeMillis();
+ if (PipeDataNodeAgent.task().getPipeCount() != 0) {
+ for (Map.Entry<String, PipeDataNodeRemainingEventAndTimeOperator> entry :
+ PipeDataNodeSinglePipeMetrics.getInstance().remainingEventAndTimeOperatorMap.entrySet()) {
+ while (entry.getValue().getRemainingNonHeartbeatEvents() > 0) {
+ if (System.currentTimeMillis() - startTime
+ > PipeConfig.getInstance().getPipeMaxWaitFinishTime()) {
+ break;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ logger.info("Interrupted when waiting for pipe to finish");
+ }
+ }
+ }
+ }
+ // Persist progress index before shutdown to accurate recovery after restart
+ PipeDataNodeAgent.task().persistAllProgressIndexLocally();
+
// Shutdown pipe progressIndex background service
PipePeriodicalJobExecutor.shutdownBackgroundService();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index f923fb2f341..6fbf870617a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -38,7 +38,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.ShutdownException;
import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.commons.file.SystemFileFactory;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.schema.ttl.TTLCache;
import org.apache.iotdb.commons.service.IService;
import org.apache.iotdb.commons.service.ServiceType;
@@ -54,7 +53,6 @@ import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
import org.apache.iotdb.db.exception.load.LoadReadOnlyException;
import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler;
@@ -219,12 +217,6 @@ public class StorageEngine implements IService {
LOGGER.info(
"Storage Engine recover cost: {}s.",
(System.currentTimeMillis() - startRecoverTime) / 1000);
-
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "StorageEngine#operateFlush",
- () -> operateFlush(new TFlushReq()),
- PipeConfig.getInstance().getPipeStorageEngineFlushTimeIntervalMs() / 1000);
},
ThreadName.STORAGE_ENGINE_RECOVER_TRIGGER.getName());
recoverEndTrigger.start();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 5e9b25bdda1..0d2737ab721 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1898,7 +1898,7 @@ public class DataRegion implements IDataRegionForQuery {
}
/** close all working tsfile processors */
- private List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
+ public List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
writeLock("asyncCloseAllWorkingTsFileProcessors");
List<Future<?>> futures = new ArrayList<>();
int count = 0;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index c4fba1fb215..992b5406708 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -227,9 +227,9 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask {
new File(targetTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath(),
new File(sourceTsFile.getPath() + ModificationFile.FILE_SUFFIX).toPath());
}
- targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
+ targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndex());
targetFile.deserialize();
- targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
+ targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndex());
}
private boolean recoverTaskInfoFromLogFile() throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index afbea98ec29..3b251a0c909 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -288,7 +288,7 @@ public class CompactionUtils {
List<TsFileResource> unseqResources) {
for (TsFileResource targetResource : targetResources) {
for (TsFileResource unseqResource : unseqResources) {
- targetResource.updateProgressIndex(unseqResource.getMaxProgressIndexAfterClose());
+ targetResource.updateProgressIndex(unseqResource.getMaxProgressIndex());
targetResource.setGeneratedByPipe(
unseqResource.isGeneratedByPipe() && targetResource.isGeneratedByPipe());
targetResource.setGeneratedByPipeConsensus(
@@ -296,7 +296,7 @@ public class CompactionUtils {
&& targetResource.isGeneratedByPipeConsensus());
}
for (TsFileResource seqResource : seqResources) {
- targetResource.updateProgressIndex(seqResource.getMaxProgressIndexAfterClose());
+ targetResource.updateProgressIndex(seqResource.getMaxProgressIndex());
targetResource.setGeneratedByPipe(
seqResource.isGeneratedByPipe() && targetResource.isGeneratedByPipe());
targetResource.setGeneratedByPipeConsensus(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index c91cb7838e3..e4f095b52c7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -321,10 +321,7 @@ public class TsFileProcessor {
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- walFlushListener.getWalEntryHandler(),
- insertRowNode,
- tsFileResource);
+ dataRegionInfo.getDataRegion().getDataRegionId(), insertRowNode, tsFileResource);
int pointInserted;
if (insertRowNode.isAligned()) {
@@ -422,10 +419,7 @@ public class TsFileProcessor {
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- walFlushListener.getWalEntryHandler(),
- insertRowsNode,
- tsFileResource);
+ dataRegionInfo.getDataRegion().getDataRegionId(), insertRowsNode, tsFileResource);
int pointInserted = 0;
for (InsertRowNode insertRowNode : insertRowsNode.getInsertRowNodeList()) {
@@ -540,10 +534,7 @@ public class TsFileProcessor {
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- walFlushListener.getWalEntryHandler(),
- insertTabletNode,
- tsFileResource);
+ dataRegionInfo.getDataRegion().getDataRegionId(), insertTabletNode, tsFileResource);
int pointInserted;
try {
@@ -1148,13 +1139,6 @@ public class TsFileProcessor {
IMemTable tmpMemTable = workMemTable == null ? new NotifyFlushMemTable() : workMemTable;
try {
- PipeInsertionDataNodeListener.getInstance()
- .listenToTsFile(
- dataRegionInfo.getDataRegion().getDataRegionId(),
- tsFileResource,
- false,
- tmpMemTable.isTotallyGeneratedByPipe());
-
// When invoke closing TsFile after insert data to memTable, we shouldn't flush until invoke
// flushing memTable in System module.
Future<?> future = addAMemtableIntoFlushingList(tmpMemTable);
@@ -1586,6 +1570,16 @@ public class TsFileProcessor {
logger.debug("Start to end file {}", tsFileResource);
}
writer.endFile();
+
+ // Listen after "endFile" to avoid unnecessary waiting for tsFile close
+ // before resource serialization to avoid missing hardlink after restart
+ PipeInsertionDataNodeListener.getInstance()
+ .listenToTsFile(
+ dataRegionInfo.getDataRegion().getDataRegionId(),
+ tsFileResource,
+ false,
+ workMemTable != null && workMemTable.isTotallyGeneratedByPipe());
+
tsFileResource.serialize();
FileTimeIndexCacheRecorder.getInstance().logFileTimeIndex(tsFileResource);
if (logger.isDebugEnabled()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
index 734c1449dfa..a9a9878627f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.dataregion.tsfile;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder;
import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager;
@@ -276,6 +277,15 @@ public class TsFileManager {
} finally {
writeUnlock();
}
+
+ PipeDataNodeResourceManager.compaction()
+ .emitResult(
+ storageGroupName,
+ dataRegionId,
+ timePartition,
+ seqFileResources,
+ unseqFileResources,
+ targetFileResources);
}
public boolean contains(TsFileResource tsFileResource, boolean sequence) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 7761ea5c3b1..ee05d2f1c74 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -1218,14 +1218,6 @@ public class TsFileResource {
.updateProgressIndex(getDataRegionId(), getTsFilePath(), maxProgressIndex);
}
- public ProgressIndex getMaxProgressIndexAfterClose() throws IllegalStateException {
- if (getStatus().equals(TsFileResourceStatus.UNCLOSED)) {
- throw new IllegalStateException(
- "Should not get progress index from a unclosing TsFileResource.");
- }
- return getMaxProgressIndex();
- }
-
public ProgressIndex getMaxProgressIndex() {
return maxProgressIndex == null ? MinimumProgressIndex.INSTANCE : maxProgressIndex;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index edc30604a6e..36c404faa7c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -342,7 +342,6 @@ public class WALBuffer extends AbstractWALBuffer {
info.metaData.add(size, searchIndex, walEntry.getMemTableId());
info.memTableId2WalDiskUsage.compute(
walEntry.getMemTableId(), (k, v) -> v == null ? size : v + size);
- walEntry.getWalFlushListener().getWalEntryHandler().setSize(size);
info.fsyncListeners.add(walEntry.getWalFlushListener());
}
@@ -593,13 +592,8 @@ public class WALBuffer extends AbstractWALBuffer {
// notify all waiting listeners
if (forceSuccess) {
- long position = lastFsyncPosition;
for (WALFlushListener fsyncListener : info.fsyncListeners) {
fsyncListener.succeed();
- if (fsyncListener.getWalEntryHandler() != null) {
- fsyncListener.getWalEntryHandler().setEntryPosition(walFileVersionId, position);
- position += fsyncListener.getWalEntryHandler().getSize();
- }
}
lastFsyncPosition = currentWALFileWriter.originalSize();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
index a8e94e2780a..18304e621b7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
@@ -77,14 +77,14 @@ public abstract class WALEntry implements SerializedSize {
} else {
throw new RuntimeException("Unknown WALEntry type");
}
- walFlushListener = new WALFlushListener(wait, value);
+ walFlushListener = new WALFlushListener(wait);
}
protected WALEntry(WALEntryType type, long memTableId, WALEntryValue value, boolean wait) {
this.type = type;
this.memTableId = memTableId;
this.value = value;
- this.walFlushListener = new WALFlushListener(wait, value);
+ this.walFlushListener = new WALFlushListener(wait);
}
public abstract void serialize(IWALByteBufferView buffer);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
index afa6651dfa4..6f801a78457 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
@@ -25,11 +25,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.ILogWriter;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,9 +175,7 @@ public class CheckpointManager implements AutoCloseable {
return;
}
memTableInfo.setFlushed();
- if (!memTableInfo.isPinned()) {
- memTableId2Info.remove(memTableId);
- }
+ memTableId2Info.remove(memTableId);
Checkpoint checkpoint =
new Checkpoint(
CheckpointType.FLUSH_MEMORY_TABLE, Collections.singletonList(memTableInfo));
@@ -261,71 +257,9 @@ public class CheckpointManager implements AutoCloseable {
// endregion
- // region methods for pipe
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable(long memTableId) throws MemTablePinException {
- infoLock.lock();
- try {
- if (!memTableId2Info.containsKey(memTableId)) {
- throw new MemTablePinException(
- String.format(
- "Fail to pin memTable-%d because this memTable doesn't exist in the wal.",
- memTableId));
- }
- MemTableInfo memTableInfo = memTableId2Info.get(memTableId);
- if (!memTableInfo.isPinned()) {
- WALInsertNodeCache.getInstance().addMemTable(memTableId);
- }
- memTableInfo.pin();
- } finally {
- infoLock.unlock();
- }
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable(long memTableId) throws MemTablePinException {
- infoLock.lock();
- try {
- if (!memTableId2Info.containsKey(memTableId)) {
- throw new MemTablePinException(
- String.format(
- "Fail to unpin memTable-%d because this memTable doesn't exist in the wal.",
- memTableId));
- }
- if (!memTableId2Info.get(memTableId).isPinned()) {
- throw new MemTablePinException(
- String.format(
- "Fail to unpin memTable-%d because this memTable hasn't been pinned.", memTableId));
- }
- MemTableInfo memTableInfo = memTableId2Info.get(memTableId);
- memTableInfo.unpin();
- if (!memTableInfo.isPinned()) {
- WALInsertNodeCache.getInstance().removeMemTable(memTableId);
- if (memTableInfo.isFlushed()) {
- memTableId2Info.remove(memTableId);
- }
- }
- } finally {
- infoLock.unlock();
- }
- }
-
- // endregion
-
- /** Get MemTableInfo of oldest unpinned MemTable, whose first version id is smallest. */
- public MemTableInfo getOldestUnpinnedMemTableInfo() {
+ public MemTableInfo getOldestMemTableInfo() {
// find oldest memTable
return activeOrPinnedMemTables().stream()
- .filter(memTableInfo -> !memTableInfo.isPinned())
.min(Comparator.comparingLong(MemTableInfo::getMemTableId))
.orElse(null);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
index 59c2a3be23b..984006b25bc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
@@ -47,8 +47,7 @@ public class MemTableInfo implements WALEntryValue {
// memTable
private IMemTable memTable;
- // memTable pin count
- private int pinCount;
+
// memTable is flushed or not
private boolean flushed;
// data region id
@@ -116,22 +115,6 @@ public class MemTableInfo implements WALEntryValue {
return memTable;
}
- public void pin() {
- this.pinCount++;
- }
-
- public void unpin() {
- this.pinCount--;
- }
-
- public boolean isPinned() {
- return pinCount > 0;
- }
-
- public int getPinCount() {
- return pinCount;
- }
-
public boolean isFlushed() {
return flushed;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
index b03b27a6994..2f257da9adc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.storageengine.dataregion.wal.io;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryPosition;
import java.io.Closeable;
import java.io.DataInputStream;
@@ -50,18 +49,6 @@ public class WALByteBufReader implements Closeable {
}
}
- public WALByteBufReader(WALEntryPosition walEntryPosition) throws IOException {
- WALInputStream walInputStream = walEntryPosition.openReadFileStream();
- try {
- this.logStream = new DataInputStream(walInputStream);
- this.metaData = walInputStream.getWALMetaData();
- this.sizeIterator = metaData.getBuffersSize().iterator();
- } catch (Exception e) {
- walInputStream.close();
- throw e;
- }
- }
-
/** Like {@link Iterator#hasNext()}. */
public boolean hasNext() {
return sizeIterator.hasNext();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
index 38b69f1162b..dfa7bf6bb9d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
@@ -41,9 +41,9 @@ public class WALFakeNode implements IWALNode {
public WALFakeNode(Status status, Exception cause) {
this.status = status;
- this.successListener = new WALFlushListener(false, null);
+ this.successListener = new WALFlushListener(false);
this.successListener.succeed();
- this.failListener = new WALFlushListener(false, null);
+ this.failListener = new WALFlushListener(false);
this.failListener.fail(cause);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
index 35432a8fd6f..003c74763da 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
@@ -49,7 +49,6 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointType;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.MemTableInfo;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALByteBufReader;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
@@ -57,7 +56,6 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.AbstractR
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.AbstractResultListener.Status;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALFlushListener;
-import org.apache.commons.lang3.StringUtils;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.utils.TsFileUtils;
import org.slf4j.Logger;
@@ -183,7 +181,6 @@ public class WALNode implements IWALNode {
buffer.write(walEntry);
// set handler for pipe
- walEntry.getWalFlushListener().getWalEntryHandler().setWalNode(this, walEntry.getMemTableId());
return walEntry.getWalFlushListener();
}
@@ -228,25 +225,6 @@ public class WALNode implements IWALNode {
// region methods for pipe
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable(long memTableId) throws MemTablePinException {
- checkpointManager.pinMemTable(memTableId);
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable(long memTableId) throws MemTablePinException {
- checkpointManager.unpinMemTable(memTableId);
- }
-
// endregion
// region Task to delete outdated .wal files
@@ -270,8 +248,6 @@ public class WALNode implements IWALNode {
// the effective information ratio
private double effectiveInfoRatio = 1.0d;
- private List<Long> pinnedMemTableIds;
-
private int fileIndexAfterFilterSafelyDeleteIndex = Integer.MAX_VALUE;
private List<Long> successfullyDeleted;
private long deleteFileSize;
@@ -297,7 +273,6 @@ public class WALNode implements IWALNode {
this.sortedWalFilesExcludingLast =
Arrays.copyOfRange(allWalFilesOfOneNode, 0, allWalFilesOfOneNode.length - 1);
this.activeOrPinnedMemTables = checkpointManager.activeOrPinnedMemTables();
- this.pinnedMemTableIds = initPinnedMemTableIds();
this.fileIndexAfterFilterSafelyDeleteIndex = initFileIndexAfterFilterSafelyDeleteIndex();
this.successfullyDeleted = new ArrayList<>();
this.deleteFileSize = 0;
@@ -318,20 +293,6 @@ public class WALNode implements IWALNode {
}
}
- private List<Long> initPinnedMemTableIds() {
- List<MemTableInfo> memTableInfos = checkpointManager.activeOrPinnedMemTables();
- if (memTableInfos.isEmpty()) {
- return new ArrayList<>();
- }
- List<Long> pinnedIds = new ArrayList<>();
- for (MemTableInfo memTableInfo : memTableInfos) {
- if (memTableInfo.isFlushed() && memTableInfo.isPinned()) {
- pinnedIds.add(memTableInfo.getMemTableId());
- }
- }
- return pinnedIds;
- }
-
@Override
public void run() {
// The intent of the loop execution here is to try to get as many memTable flush or snapshot
@@ -365,7 +326,7 @@ public class WALNode implements IWALNode {
private void updateEffectiveInfoRationAndUpdateMetric() {
// calculate effective information ratio
long costOfActiveMemTables = checkpointManager.getTotalCostOfActiveMemTables();
- MemTableInfo oldestUnpinnedMemTableInfo = checkpointManager.getOldestUnpinnedMemTableInfo();
+ MemTableInfo oldestUnpinnedMemTableInfo = checkpointManager.getOldestMemTableInfo();
long avgFileSize =
getFileNum() != 0
? getDiskUsage() / getFileNum()
@@ -389,45 +350,10 @@ public class WALNode implements IWALNode {
}
private void summarizeExecuteResult() {
- if (!pinnedMemTableIds.isEmpty()
- || fileIndexAfterFilterSafelyDeleteIndex < sortedWalFilesExcludingLast.length) {
- if (logger.isDebugEnabled()) {
- StringBuilder summary =
- new StringBuilder(
- String.format(
- "wal node-%s delete outdated files summary:the range is: [%d,%d], delete successful is [%s], safely delete file index is: [%s].The following reasons influenced the result: %s",
- identifier,
- WALFileUtils.parseVersionId(sortedWalFilesExcludingLast[0].getName()),
- WALFileUtils.parseVersionId(
- sortedWalFilesExcludingLast[sortedWalFilesExcludingLast.length - 1]
- .getName()),
- StringUtils.join(successfullyDeleted, ","),
- fileIndexAfterFilterSafelyDeleteIndex,
- System.lineSeparator()));
-
- if (!pinnedMemTableIds.isEmpty()) {
- summary
- .append("- MemTable has been flushed but pinned by PIPE, the MemTableId list is : ")
- .append(StringUtils.join(pinnedMemTableIds, ","))
- .append(".")
- .append(System.lineSeparator());
- }
- if (fileIndexAfterFilterSafelyDeleteIndex < sortedWalFilesExcludingLast.length) {
- summary.append(
- String.format(
- "- The data in the wal file was not consumed by the consensus group,current search index is %d, safely delete index is %d",
- getCurrentSearchIndex(), safelyDeletedSearchIndex));
- }
- String summaryLog = summary.toString();
- logger.debug(summaryLog);
- }
-
- } else {
- logger.debug(
- "Successfully delete {} outdated wal files for wal node-{}",
- successfullyDeleted.size(),
- identifier);
- }
+ logger.debug(
+ "Successfully delete {} outdated wal files for wal node-{}",
+ successfullyDeleted.size(),
+ identifier);
}
/** Delete obsolete wal files while recording which succeeded or failed */
@@ -475,20 +401,10 @@ public class WALNode implements IWALNode {
return false;
}
// find oldest memTable
- MemTableInfo oldestMemTableInfo = checkpointManager.getOldestUnpinnedMemTableInfo();
+ MemTableInfo oldestMemTableInfo = checkpointManager.getOldestMemTableInfo();
if (oldestMemTableInfo == null) {
return false;
}
- if (oldestMemTableInfo.isPinned()) {
- logger.warn(
- "Pipe: Effective information ratio {} of wal node-{} is below wal min effective info ratio {}. But fail to delete memTable-{}'s wal files because they are pinned by the Pipe module. Pin count: {}.",
- effectiveInfoRatio,
- identifier,
- config.getWalMinEffectiveInfoRatio(),
- oldestMemTableInfo.getMemTableId(),
- oldestMemTableInfo.getPinCount());
- return false;
- }
IMemTable oldestMemTable = oldestMemTableInfo.getMemTable();
if (oldestMemTable == null) {
return false;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
deleted file mode 100644
index f5d7406f5a6..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
-
-import org.apache.tsfile.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * This handler is used by the Pipe to find the corresponding {@link InsertNode}. Besides, it can
- * try to pin/unpin the {@link WALEntry}s by the memTable id.
- */
-public class WALEntryHandler {
-
- private static final Logger logger = LoggerFactory.getLogger(WALEntryHandler.class);
-
- private long memTableId = -1;
-
- // cached value, null after this value is flushed to wal successfully
- @SuppressWarnings("squid:S3077")
- private volatile WALEntryValue value;
-
- // wal entry's position in the wal, valid after the value is flushed to wal successfully
- // it's safe to use volatile here to make this reference thread-safe.
- @SuppressWarnings("squid:S3077")
- private final WALEntryPosition walEntryPosition = new WALEntryPosition();
-
- // wal node, null when wal is disabled
- private WALNode walNode = null;
-
- private volatile boolean isHardlink = false;
- private final AtomicReference<File> hardlinkFile = new AtomicReference<>();
-
- public WALEntryHandler(final WALEntryValue value) {
- this.value = value;
- }
-
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable() throws MemTablePinException {
- if (walNode == null || memTableId < 0) {
- throw new MemTablePinException("Fail to pin memTable because of internal error.");
- }
- walNode.pinMemTable(memTableId);
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable() throws MemTablePinException {
- if (walNode == null || memTableId < 0) {
- throw new MemTablePinException("Fail to pin memTable because of internal error.");
- }
- walNode.unpinMemTable(memTableId);
- }
-
- public InsertNode getInsertNodeViaCacheIfPossible() {
- try {
- final WALEntryValue finalValue = value;
- if (finalValue instanceof InsertNode) {
- return (InsertNode) finalValue;
- }
- final Pair<ByteBuffer, InsertNode> byteBufferInsertNodePair =
- walEntryPosition.getByteBufferOrInsertNodeIfPossible();
- return byteBufferInsertNodePair == null ? null : byteBufferInsertNodePair.getRight();
- } catch (final Exception e) {
- logger.warn("Fail to get insert node via cache. {}", this, e);
- throw e;
- }
- }
-
- /**
- * Get this handler's value.
- *
- * @throws WALPipeException when failing to get the value.
- */
- public InsertNode getInsertNode() throws WALPipeException {
- // return local cache
- final WALEntryValue res = value;
- if (res != null) {
- if (res instanceof InsertNode) {
- return (InsertNode) res;
- } else {
- throw new WALPipeException("Fail to get value because the entry type isn't InsertNode.");
- }
- }
-
- // wait until the position is ready
- while (!walEntryPosition.canRead()) {
- try {
- synchronized (this) {
- this.wait();
- }
- } catch (final InterruptedException e) {
- logger.warn("Interrupted when waiting for result.", e);
- Thread.currentThread().interrupt();
- }
- }
-
- final InsertNode node = isHardlink ? readFromHardlinkWALFile() : readFromOriginalWALFile();
- if (node == null) {
- throw new WALPipeException(
- String.format("Fail to get the wal value of the position %s.", walEntryPosition));
- }
- return node;
- }
-
- public ByteBuffer getByteBuffer() throws WALPipeException {
- // wait until the position is ready
- while (!walEntryPosition.canRead()) {
- try {
- synchronized (this) {
- this.wait();
- }
- } catch (InterruptedException e) {
- logger.warn("Interrupted when waiting for result.", e);
- Thread.currentThread().interrupt();
- }
- }
-
- final ByteBuffer buffer = readByteBufferFromWALFile();
- if (buffer == null) {
- throw new WALPipeException(
- String.format("Fail to get the wal value of the position %s.", walEntryPosition));
- }
- return buffer;
- }
-
- private InsertNode readFromOriginalWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readInsertNodeViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- private InsertNode readFromHardlinkWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readInsertNodeViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- private ByteBuffer readByteBufferFromWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readByteBufferViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- public void setWalNode(final WALNode walNode, final long memTableId) {
- this.walNode = walNode;
- this.memTableId = memTableId;
- walEntryPosition.setWalNode(walNode, memTableId);
- }
-
- public long getMemTableId() {
- return memTableId;
- }
-
- public void setEntryPosition(final long walFileVersionId, final long position) {
- this.walEntryPosition.setEntryPosition(walFileVersionId, position, value);
- this.value = null;
- synchronized (this) {
- this.notifyAll();
- }
- }
-
- public WALEntryPosition getWalEntryPosition() {
- return walEntryPosition;
- }
-
- public int getSize() {
- return walEntryPosition.getSize();
- }
-
- public void setSize(final int size) {
- this.walEntryPosition.setSize(size);
- }
-
- public void hardlinkTo(File hardlinkFile) {
- isHardlink = true;
- this.hardlinkFile.set(hardlinkFile);
- }
-
- @Override
- public String toString() {
- return "WALEntryHandler{"
- + "memTableId="
- + memTableId
- + ", value="
- + value
- + ", walEntryPosition="
- + walEntryPosition
- + '}';
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
deleted file mode 100644
index 4d71cb1030c..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALInputStream;
-import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
-
-import org.apache.tsfile.utils.Pair;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.StandardOpenOption;
-import java.util.Objects;
-
-/**
- * This class uses the tuple(identifier, file, position) to denote the position of the wal entry,
- * and give some methods to read the content from the disk.
- */
-public class WALEntryPosition {
- private volatile String identifier = "";
- private volatile long walFileVersionId = -1;
- private volatile long position;
- private volatile int size;
- // wal node, null when wal is disabled
- private WALNode walNode = null;
- // wal file is not null when openReadFileChannel method has been called
- private File walFile = null;
- // cache for wal entry
- private WALInsertNodeCache cache = null;
-
- private static final String ENTRY_NOT_READY_MESSAGE = "This entry isn't ready for read.";
-
- public WALEntryPosition() {}
-
- public WALEntryPosition(String identifier, long walFileVersionId, long position, int size) {
- this.identifier = identifier;
- this.walFileVersionId = walFileVersionId;
- this.position = position;
- this.size = size;
- }
-
- /**
- * Try to read the wal entry directly from the cache. No need to check if the wal entry is ready
- * for read.
- */
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNodeIfPossible() {
- return cache.getByteBufferOrInsertNodeIfPossible(this);
- }
-
- /**
- * Read the wal entry and parse it to the InsertNode. Use LRU cache to accelerate read.
- *
- * @throws IOException failing to read.
- */
- public InsertNode readInsertNodeViaCacheAfterCanRead() throws IOException {
- if (!canRead()) {
- throw new IOException(ENTRY_NOT_READY_MESSAGE);
- }
- return cache.getInsertNode(this);
- }
-
- /**
- * Read the wal entry and get the raw bytebuffer. Use LRU cache to accelerate read.
- *
- * @throws IOException failing to read.
- */
- public ByteBuffer readByteBufferViaCacheAfterCanRead() throws IOException {
- if (!canRead()) {
- throw new IOException(ENTRY_NOT_READY_MESSAGE);
- }
- return cache.getByteBuffer(this);
- }
-
- /**
- * Read the byte buffer directly.
- *
- * @throws IOException failing to read.
- */
- ByteBuffer read() throws IOException {
- if (!canRead()) {
- throw new IOException("Target file hasn't been specified.");
- }
- // TODO: Reuse the file stream
- try (WALInputStream is = openReadFileStream()) {
- is.skipToGivenLogicalPosition(position);
- ByteBuffer buffer = ByteBuffer.allocate(size);
- is.read(buffer);
- return buffer;
- }
- }
-
- /**
- * Open the read file channel for this wal entry, this method will retry automatically when the
- * file is sealed when opening the file channel.
- *
- * @throws IOException failing to open the file channel.
- */
- public FileChannel openReadFileChannel() throws IOException {
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } else {
- try {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } catch (IOException e) {
- // unsealed file may be renamed after sealed, so we should try again
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } else {
- throw e;
- }
- }
- }
- }
-
- public WALInputStream openReadFileStream() throws IOException {
- // TODO: Refactor this part of code
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } else {
- try {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } catch (IOException e) {
- // unsealed file may be renamed after sealed, so we should try again
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } else {
- throw e;
- }
- }
- }
- }
-
- public File getWalFile() {
- return walFile;
- }
-
- /** Return true only when the tuple(file, position, size) is ready. */
- public boolean canRead() {
- return walFileVersionId >= 0;
- }
-
- /** Return true only when this wal file is sealed. */
- public boolean isInSealedFile() {
- if (walNode == null || !canRead()) {
- throw new RuntimeException(ENTRY_NOT_READY_MESSAGE);
- }
- return walFileVersionId < walNode.getCurrentWALFileVersion();
- }
-
- public void setWalNode(WALNode walNode, long memTableId) {
- this.walNode = walNode;
- identifier = walNode.getIdentifier();
- cache = WALInsertNodeCache.getInstance();
- }
-
- public String getIdentifier() {
- return identifier;
- }
-
- public void setEntryPosition(long walFileVersionId, long position, WALEntryValue value) {
- this.position = position;
- this.walFileVersionId = walFileVersionId;
- if (cache != null && value instanceof InsertNode) {
- cache.cacheInsertNodeIfNeeded(this, (InsertNode) value);
- }
- }
-
- public long getPosition() {
- return position;
- }
-
- public long getWalFileVersionId() {
- return walFileVersionId;
- }
-
- public void setSize(int size) {
- this.size = size;
- }
-
- public int getSize() {
- return size;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(identifier, walFileVersionId, position);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- WALEntryPosition that = (WALEntryPosition) o;
- return identifier.equals(that.identifier)
- && walFileVersionId == that.walFileVersionId
- && position == that.position;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
deleted file mode 100644
index f69dc4a0826..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlockType;
-import org.apache.iotdb.db.pipe.resource.memory.PipeModelFixedMemoryBlock;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType;
-import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALByteBufReader;
-
-import com.github.benmanes.caffeine.cache.CacheLoader;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import com.github.benmanes.caffeine.cache.Weigher;
-import org.apache.tsfile.utils.Pair;
-import org.checkerframework.checker.nullness.qual.NonNull;
-import org.checkerframework.checker.nullness.qual.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/** This cache is used by {@link WALEntryPosition}. */
-public class WALInsertNodeCache {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WALInsertNodeCache.class);
- private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
-
- private static PipeModelFixedMemoryBlock walModelFixedMemory = null;
-
- // LRU cache, find Pair<ByteBuffer, InsertNode> by WALEntryPosition
- private final LoadingCache<WALEntryPosition, Pair<ByteBuffer, InsertNode>> lruCache;
-
- // ids of all pinned memTables
- private final Set<Long> memTablesNeedSearch = ConcurrentHashMap.newKeySet();
-
- private volatile boolean hasPipeRunning = false;
-
- private WALInsertNodeCache() {
- if (walModelFixedMemory == null) {
- init();
- }
-
- final long requestedAllocateSize =
- (long)
- (PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- * PIPE_CONFIG.getPipeDataStructureWalMemoryProportion());
-
- lruCache =
- Caffeine.newBuilder()
- .maximumWeight(requestedAllocateSize)
- .weigher(
- (Weigher<WALEntryPosition, Pair<ByteBuffer, InsertNode>>)
- (position, pair) -> {
- long weightInLong = 0L;
- if (pair.right != null) {
- weightInLong = InsertNodeMemoryEstimator.sizeOf(pair.right);
- } else {
- weightInLong = position.getSize();
- }
- if (weightInLong <= 0) {
- return Integer.MAX_VALUE;
- }
- final int weightInInt = (int) weightInLong;
- return weightInInt != weightInLong ? Integer.MAX_VALUE : weightInInt;
- })
- .recordStats()
- .build(new WALInsertNodeCacheLoader());
- }
-
- // please call this method at PipeLauncher
- public static void init() {
- if (walModelFixedMemory != null) {
- return;
- }
- try {
- // Allocate memory for the fixed memory block of WAL
- walModelFixedMemory =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(
- (long)
- (PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- * PIPE_CONFIG.getPipeDataStructureWalMemoryProportion()),
- PipeMemoryBlockType.WAL);
- } catch (Exception e) {
- LOGGER.error("Failed to initialize WAL model fixed memory block", e);
- walModelFixedMemory =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(0, PipeMemoryBlockType.WAL);
- }
- }
-
- /////////////////////////// Getter & Setter ///////////////////////////
-
- public InsertNode getInsertNode(final WALEntryPosition position) {
- final Pair<ByteBuffer, InsertNode> pair = getByteBufferOrInsertNode(position);
-
- if (pair.getRight() != null) {
- return pair.getRight();
- }
-
- if (pair.getLeft() == null) {
- throw new IllegalStateException();
- }
-
- try {
- // multi pipes may share the same wal entry, so we need to wrap the byte[] into
- // different ByteBuffer for each pipe
- final InsertNode insertNode = parse(ByteBuffer.wrap(pair.getLeft().array()));
- pair.setRight(insertNode);
- return insertNode;
- } catch (final Exception e) {
- LOGGER.error(
- "Parsing failed when recovering insertNode from wal, walFile:{}, position:{}, size:{}, exception:",
- position.getWalFile(),
- position.getPosition(),
- position.getSize(),
- e);
- throw e;
- }
- }
-
- private InsertNode parse(final ByteBuffer buffer) {
- final PlanNode node = WALEntry.deserializeForConsensus(buffer);
- if (node instanceof InsertNode) {
- return (InsertNode) node;
- } else {
- return null;
- }
- }
-
- public ByteBuffer getByteBuffer(final WALEntryPosition position) {
- Pair<ByteBuffer, InsertNode> pair = getByteBufferOrInsertNode(position);
-
- if (pair.getLeft() != null) {
- // multi pipes may share the same wal entry, so we need to wrap the byte[] into
- // different ByteBuffer for each pipe
- return ByteBuffer.wrap(pair.getLeft().array());
- }
-
- // forbid multi threads to invalidate and load the same entry
- synchronized (this) {
- lruCache.invalidate(position);
- pair = getByteBufferOrInsertNode(position);
- }
-
- if (pair.getLeft() == null) {
- throw new IllegalStateException();
- }
-
- return ByteBuffer.wrap(pair.getLeft().array());
- }
-
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNode(final WALEntryPosition position) {
- hasPipeRunning = true;
-
- final Pair<ByteBuffer, InsertNode> pair = lruCache.get(position);
-
- if (pair == null) {
- throw new IllegalStateException();
- }
-
- return pair;
- }
-
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNodeIfPossible(
- final WALEntryPosition position) {
- hasPipeRunning = true;
- return lruCache.getIfPresent(position);
- }
-
- public void cacheInsertNodeIfNeeded(
- final WALEntryPosition walEntryPosition, final InsertNode insertNode) {
- // reduce memory usage
- if (hasPipeRunning) {
- lruCache.put(walEntryPosition, new Pair<>(null, insertNode));
- }
- }
-
- //////////////////////////// APIs provided for metric framework ////////////////////////////
-
- public double getCacheHitRate() {
- return Objects.nonNull(lruCache) ? lruCache.stats().hitRate() : 0;
- }
-
- public double getCacheHitCount() {
- return Objects.nonNull(lruCache) ? lruCache.stats().hitCount() : 0;
- }
-
- public double getCacheRequestCount() {
- return Objects.nonNull(lruCache) ? lruCache.stats().requestCount() : 0;
- }
-
- /////////////////////////// MemTable ///////////////////////////
-
- public void addMemTable(final long memTableId) {
- memTablesNeedSearch.add(memTableId);
- }
-
- public void removeMemTable(final long memTableId) {
- memTablesNeedSearch.remove(memTableId);
- }
-
- /////////////////////////// Cache Loader ///////////////////////////
-
- class WALInsertNodeCacheLoader
- implements CacheLoader<WALEntryPosition, Pair<ByteBuffer, InsertNode>> {
-
- @Override
- public @Nullable Pair<ByteBuffer, InsertNode> load(@NonNull final WALEntryPosition key)
- throws Exception {
- return new Pair<>(key.read(), null);
- }
-
- /** Batch load all wal entries in the file when any one key is absent. */
- @Override
- public @NonNull Map<@NonNull WALEntryPosition, @NonNull Pair<ByteBuffer, InsertNode>> loadAll(
- @NonNull final Iterable<? extends @NonNull WALEntryPosition> walEntryPositions) {
- final Map<WALEntryPosition, Pair<ByteBuffer, InsertNode>> loadedEntries = new HashMap<>();
-
- for (final WALEntryPosition walEntryPosition : walEntryPositions) {
- if (loadedEntries.containsKey(walEntryPosition) || !walEntryPosition.canRead()) {
- continue;
- }
-
- final long walFileVersionId = walEntryPosition.getWalFileVersionId();
-
- // load one when wal file is not sealed
- if (!walEntryPosition.isInSealedFile()) {
- try {
- loadedEntries.put(walEntryPosition, load(walEntryPosition));
- } catch (final Exception e) {
- LOGGER.info(
- "Fail to cache wal entries from the wal file with version id {}",
- walFileVersionId,
- e);
- }
- continue;
- }
-
- // batch load when wal file is sealed
- long position = 0;
- try (final WALByteBufReader walByteBufReader = new WALByteBufReader(walEntryPosition)) {
- while (walByteBufReader.hasNext()) {
- // see WALInfoEntry#serialize, entry type + memtable id + plan node type
- final ByteBuffer buffer = walByteBufReader.next();
-
- final int size = buffer.capacity();
- final WALEntryType type = WALEntryType.valueOf(buffer.get());
- final long memTableId = buffer.getLong();
-
- if ((memTablesNeedSearch.contains(memTableId)
- || walEntryPosition.getPosition() == position)
- && type.needSearch()) {
- buffer.clear();
- loadedEntries.put(
- new WALEntryPosition(
- walEntryPosition.getIdentifier(), walFileVersionId, position, size),
- new Pair<>(buffer, null));
- }
-
- position += size;
- }
- } catch (final IOException e) {
- LOGGER.info(
- "Fail to cache wal entries from the wal file with version id {}",
- walFileVersionId,
- e);
- }
- }
-
- return loadedEntries;
- }
- }
-
- /////////////////////////// Singleton ///////////////////////////
-
- public static WALInsertNodeCache getInstance() {
- return InstanceHolder.INSTANCE;
- }
-
- private static class InstanceHolder {
-
- public static final WALInsertNodeCache INSTANCE = new WALInsertNodeCache();
-
- private InstanceHolder() {
- // forbidding instantiation
- }
- }
-
- /////////////////////////// Test Only ///////////////////////////
-
- @TestOnly
- boolean contains(WALEntryPosition position) {
- return lruCache.getIfPresent(position) != null;
- }
-
- @TestOnly
- public void clear() {
- lruCache.invalidateAll();
- memTablesNeedSearch.clear();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
index 8c84a0cb0fe..7896ea01dcf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
@@ -19,20 +19,10 @@
package org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
/** This class helps judge whether wal is flushed to the storage device. */
public class WALFlushListener extends AbstractResultListener {
- // handler for pipeline, only exists when value is InsertNode
- private final WALEntryHandler walEntryHandler;
- public WALFlushListener(boolean wait, WALEntryValue value) {
+ public WALFlushListener(boolean wait) {
super(wait);
- walEntryHandler = new WALEntryHandler(value);
- }
-
- public WALEntryHandler getWalEntryHandler() {
- return walEntryHandler;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
index 7dfe91ad62e..cb147f9c286 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
@@ -20,9 +20,6 @@
package org.apache.iotdb.db.storageengine.load.converter;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
-import org.apache.iotdb.commons.concurrent.ThreadName;
-import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBPipePattern;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReq;
@@ -48,14 +45,6 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil.calculateTabletSizeInBytes;
@@ -70,9 +59,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
.getConfig()
.getLoadTsFileTabletConversionBatchMemorySizeInBytes();
- private static final AtomicReference<WrappedThreadPoolExecutor> executorPool =
- new AtomicReference<>();
-
private final StatementExecutor statementExecutor;
@FunctionalInterface
@@ -80,21 +66,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
TSStatus execute(final Statement statement);
}
- public static class CallerBlocksPolicy implements RejectedExecutionHandler {
- public CallerBlocksPolicy() {}
-
- public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
- if (!e.isShutdown()) {
- try {
- e.getQueue().put(r);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new RejectedExecutionException("task " + r + " rejected from " + e, ie);
- }
- }
- }
- }
-
public LoadTreeStatementDataTypeConvertExecutionVisitor(
final StatementExecutor statementExecutor) {
this.statementExecutor = statementExecutor;
@@ -118,7 +89,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
final List<Long> tabletRawReqSizes = new ArrayList<>();
try {
- final List<Future<TSStatus>> executionFutures = new ArrayList<>();
for (final File file : loadTsFileStatement.getTsFiles()) {
try (final TsFileInsertionScanDataContainer container =
new TsFileInsertionScanDataContainer(
@@ -136,16 +106,9 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
continue;
}
- final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
- batchStatement.setInsertTabletStatementList(
- tabletRawReqs.stream()
- .map(
- req ->
- new LoadConvertedInsertTabletStatement(
- req.constructStatement(),
- loadTsFileStatement.isConvertOnTypeMismatch()))
- .collect(Collectors.toList()));
- executionFutures.add(executeInsertMultiTabletsWithRetry(batchStatement));
+ final TSStatus result =
+ executeInsertMultiTabletsWithRetry(
+ tabletRawReqs, loadTsFileStatement.isConvertOnTypeMismatch());
for (final long memoryCost : tabletRawReqSizes) {
block.reduceMemoryUsage(memoryCost);
@@ -153,6 +116,10 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
tabletRawReqs.clear();
tabletRawReqSizes.clear();
+ if (!handleTSStatus(result, loadTsFileStatement)) {
+ return Optional.empty();
+ }
+
tabletRawReqs.add(tabletRawReq);
tabletRawReqSizes.add(curMemory);
block.addMemoryUsage(curMemory);
@@ -166,36 +133,22 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
if (!tabletRawReqs.isEmpty()) {
try {
- final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
- batchStatement.setInsertTabletStatementList(
- tabletRawReqs.stream()
- .map(
- req ->
- new LoadConvertedInsertTabletStatement(
- req.constructStatement(),
- loadTsFileStatement.isConvertOnTypeMismatch()))
- .collect(Collectors.toList()));
- executionFutures.add(executeInsertMultiTabletsWithRetry(batchStatement));
+ final TSStatus result =
+ executeInsertMultiTabletsWithRetry(
+ tabletRawReqs, loadTsFileStatement.isConvertOnTypeMismatch());
for (final long memoryCost : tabletRawReqSizes) {
block.reduceMemoryUsage(memoryCost);
}
tabletRawReqs.clear();
tabletRawReqSizes.clear();
- } catch (final Exception e) {
- LOGGER.warn(
- "Failed to convert data type for LoadTsFileStatement: {}.", loadTsFileStatement, e);
- return Optional.empty();
- }
- }
- for (final Future<TSStatus> future : executionFutures) {
- try {
- if (!handleTSStatus(future.get(), loadTsFileStatement)) {
+ if (!handleTSStatus(result, loadTsFileStatement)) {
return Optional.empty();
}
- } catch (ExecutionException | InterruptedException e) {
- LOGGER.warn("Exception occurs when executing insertion during tablet conversion: ", e);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to convert data type for LoadTsFileStatement: {}.", loadTsFileStatement, e);
return Optional.empty();
}
}
@@ -226,67 +179,43 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
return Optional.of(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
- private Future<TSStatus> executeInsertMultiTabletsWithRetry(
- final InsertMultiTabletsStatement batchStatement) {
- return getExecutorPool()
- .submit(
- () -> {
- TSStatus result;
- try {
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(batchStatement));
-
- // Retry max 5 times if the write process is rejected
- for (int i = 0;
- i < 5
- && result.getCode()
- == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
- i++) {
- Thread.sleep(100L * (i + 1));
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(batchStatement));
- }
- } catch (final Exception e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
- }
- return result;
- });
- }
-
- public static WrappedThreadPoolExecutor getExecutorPool() {
- if (executorPool.get() == null) {
- synchronized (executorPool) {
- if (executorPool.get() == null) {
- executorPool.set(
- new WrappedThreadPoolExecutor(
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount(),
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount(),
- 0L,
- TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount()),
- new IoTThreadFactory(ThreadName.LOAD_DATATYPE_CONVERT_POOL.getName()),
- ThreadName.LOAD_DATATYPE_CONVERT_POOL.getName(),
- new CallerBlocksPolicy()));
- }
+ private TSStatus executeInsertMultiTabletsWithRetry(
+ final List<PipeTransferTabletRawReq> tabletRawReqs, boolean isConvertOnTypeMismatch) {
+ final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
+ batchStatement.setInsertTabletStatementList(
+ tabletRawReqs.stream()
+ .map(
+ req ->
+ new LoadConvertedInsertTabletStatement(
+ req.constructStatement(), isConvertOnTypeMismatch))
+ .collect(Collectors.toList()));
+
+ TSStatus result;
+ try {
+ result =
+ batchStatement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(batchStatement));
+
+ // Retry max 5 times if the write process is rejected
+ for (int i = 0;
+ i < 5
+ && result.getCode()
+ == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
+ i++) {
+ Thread.sleep(100L * (i + 1));
+ result =
+ batchStatement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(batchStatement));
+ }
+ } catch (final Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
}
+ result = batchStatement.accept(LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
}
- return executorPool.get();
+ return result;
}
private static boolean handleTSStatus(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
index a46546e3e36..ee2a8fe2547 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
@@ -20,11 +20,7 @@
package org.apache.iotdb.db.storageengine.load.converter;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.protocol.session.IClientSession;
-import org.apache.iotdb.db.protocol.session.InternalClientSession;
import org.apache.iotdb.db.protocol.session.SessionManager;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
@@ -37,7 +33,6 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.ZoneId;
import java.util.Optional;
public class LoadTsFileDataTypeConverter {
@@ -73,32 +68,17 @@ public class LoadTsFileDataTypeConverter {
}
private TSStatus executeForTreeModel(final Statement statement) {
- final IClientSession session =
- new InternalClientSession(
- String.format(
- "%s_%s",
- LoadTsFileDataTypeConverter.class.getSimpleName(),
- Thread.currentThread().getName()));
- session.setUsername(AuthorityChecker.SUPER_USER);
- session.setClientVersion(IoTDBConstant.ClientVersion.V_1_0);
- session.setZoneId(ZoneId.systemDefault());
-
- SESSION_MANAGER.registerSession(session);
- try {
- return Coordinator.getInstance()
- .executeForTreeModel(
- isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement,
- SESSION_MANAGER.requestQueryId(),
- SESSION_MANAGER.getSessionInfo(session),
- "",
- ClusterPartitionFetcher.getInstance(),
- ClusterSchemaFetcher.getInstance(),
- IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(),
- false)
- .status;
- } finally {
- SESSION_MANAGER.removeCurrSession();
- }
+ return Coordinator.getInstance()
+ .executeForTreeModel(
+ isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement,
+ SESSION_MANAGER.requestQueryId(),
+ SESSION_MANAGER.getSessionInfo(SESSION_MANAGER.getCurrSession()),
+ "",
+ ClusterPartitionFetcher.getInstance(),
+ ClusterSchemaFetcher.getInstance(),
+ IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(),
+ false)
+ .status;
}
public boolean isSuccessful(final TSStatus status) {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
index 8e69473b95c..a581d111dc6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
@@ -71,7 +71,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getTimestampPrecision(), deserializeReq.getTimestampPrecision());
}
@@ -94,7 +93,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getInsertNode(), deserializeReq.getInsertNode());
@@ -114,7 +112,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
}
@Test
@@ -137,7 +134,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getPlanNode(), deserializeReq.getPlanNode());
}
@@ -176,7 +172,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
final Statement statement =
req.constructStatement(); // will call PipeTransferTabletRawReq.sortTablet() here
@@ -264,8 +259,6 @@ public class PipeDataNodeThriftRequestTest {
final PipeTransferTabletBatchReq deserializedReq =
PipeTransferTabletBatchReq.fromTPipeTransferReq(req);
- Assert.assertArrayEquals(
- new byte[] {'a', 'b'}, deserializedReq.getBinaryReqs().get(0).getBody());
Assert.assertEquals(node, deserializedReq.getInsertNodeReqs().get(0).getInsertNode());
Assert.assertEquals(t, deserializedReq.getTabletReqs().get(0).getTablet());
Assert.assertFalse(deserializedReq.getTabletReqs().get(0).getIsAligned());
@@ -283,7 +276,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
... 3852 lines suppressed ...