You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ju...@apache.org on 2025/07/17 10:12:05 UTC
(iotdb) branch master updated: Pipe: Fixed multiple bugs (#15964)
This is an automated email from the ASF dual-hosted git repository.
justinchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 33eec34e837 Pipe: Fixed multiple bugs (#15964)
33eec34e837 is described below
commit 33eec34e83764cf837215dee859522fd0e8561be
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Thu Jul 17 18:12:01 2025 +0800
Pipe: Fixed multiple bugs (#15964)
* [To dev/1.3] Fixed multiple bugs of pipe
* spotless
* Partial
* Update PipeInsertNodeTabletInsertionEvent.java
* partial
* partial
* fixed-partial
* part
* fix-compile
* Update TsFileInsertionEventParserTest.java
* Update IoTDBLoadTsFileIT.java
* Update IoTDBLoadTsFileIT.java
---
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 3 +
.../manual/AbstractPipeTableModelDualManualIT.java | 6 +-
.../treemodel/auto/basic/IoTDBPipeExtractorIT.java | 6 +-
.../treemodel/auto/basic/IoTDBPipeProcessorIT.java | 6 +-
.../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 6 +-
.../auto/enhanced/IoTDBPipeAutoConflictIT.java | 6 +-
.../auto/enhanced/IoTDBPipeClusterIT.java | 6 +-
.../enhanced/IoTDBPipeConnectorCompressionIT.java | 7 +-
.../auto/enhanced/IoTDBPipeIdempotentIT.java | 6 +-
.../auto/enhanced/IoTDBPipeWithLoadIT.java | 6 +-
.../manual/AbstractPipeDualTreeModelManualIT.java | 6 +-
.../manual/IoTDBPipeTypeConversionISessionIT.java | 12 +-
.../manual/IoTDBPipeTypeConversionIT.java | 2 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 1 +
.../it/cluster/IoTDBSubscriptionRestartIT.java | 3 +-
.../it/dual/AbstractSubscriptionDualIT.java | 3 +
.../IoTDBSubscriptionConsumerGroupIT.java | 3 +
.../IoTDBSubscriptionTimePrecisionIT.java | 2 +
.../dual/treemodel/IoTDBSubscriptionTopicIT.java | 3 +
.../it/local/AbstractSubscriptionLocalIT.java | 7 +-
.../it/triple/AbstractSubscriptionTripleIT.java | 5 +-
.../apache/iotdb/tools/it/ExportTsFileTestIT.java | 6 +-
.../event/dml/insertion/TsFileInsertionEvent.java | 10 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../agent/task/PipeConfigNodeSubtaskExecutor.java | 4 +-
.../pipe/agent/task/PipeConfigNodeTaskAgent.java | 8 +
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../connector/PipeConfigNodeThriftRequestTest.java | 4 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 20 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 +-
.../db/pipe/agent/runtime/PipeAgentLauncher.java | 2 -
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 10 -
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 547 +++++++++----------
.../task/builder/PipeDataNodeTaskBuilder.java | 23 +-
.../execution/PipeConnectorSubtaskExecutor.java | 8 +-
.../execution/PipeProcessorSubtaskExecutor.java | 2 +-
.../task/execution/PipeSubtaskExecutorManager.java | 18 +-
.../agent/task/stage/PipeTaskConnectorStage.java | 6 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 5 +
.../connector/PipeConnectorSubtaskManager.java | 19 +-
.../PipeRealtimePriorityBlockingQueue.java | 151 +++++-
.../client/IoTDBDataNodeAsyncClientManager.java | 36 ++
.../evolvable/batch/PipeTabletEventBatch.java | 39 +-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 10 +-
.../batch/PipeTabletEventTsFileBatch.java | 7 +
.../evolvable/request/PipeTransferPlanNodeReq.java | 1 -
.../request/PipeTransferTabletBatchReq.java | 1 -
.../request/PipeTransferTabletBinaryReq.java | 1 -
.../request/PipeTransferTabletInsertNodeReq.java | 1 -
.../request/PipeTransferTabletRawReq.java | 1 -
.../airgap/IoTDBDataRegionAirGapConnector.java | 3 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 3 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 2 +-
.../PipeConsensusTransferBatchReqBuilder.java | 3 +-
.../async/IoTDBDataRegionAsyncConnector.java | 91 ++--
.../async/handler/PipeTransferTsFileHandler.java | 45 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 3 +-
.../protocol/writeback/WriteBackConnector.java | 3 +-
.../consensus/PipeConsensusSubtaskExecutor.java | 6 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 101 ++--
.../event/common/terminate/PipeTerminateEvent.java | 44 --
.../tsfile/PipeCompactedTsFileInsertionEvent.java | 264 ++++++++++
.../common/tsfile/PipeTsFileInsertionEvent.java | 98 ++--
.../parser/TsFileInsertionEventParserProvider.java | 4 +-
.../event/realtime/PipeRealtimeEventFactory.java | 12 +-
.../dataregion/DataRegionWatermarkInjector.java | 4 -
.../dataregion/IoTDBDataRegionExtractor.java | 107 ++--
...oricalDataRegionTsFileAndDeletionExtractor.java | 45 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 35 ++
.../PipeRealtimeDataRegionHybridExtractor.java | 180 ++-----
.../PipeRealtimeDataRegionLogExtractor.java | 5 +-
.../PipeRealtimeDataRegionTsFileExtractor.java | 7 +
.../realtime/assigner/PipeDataRegionAssigner.java | 50 +-
.../PipeTsFileEpochProgressIndexKeeper.java | 30 +-
.../dataregion/realtime/epoch/TsFileEpoch.java | 17 +-
.../realtime/epoch/TsFileEpochManager.java | 4 +-
.../listener/PipeInsertionDataNodeListener.java | 8 +-
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 3 -
.../PipeDataNodeRemainingEventAndTimeOperator.java | 22 +-
.../overview/PipeDataNodeSinglePipeMetrics.java | 84 ++-
.../pipe/metric/overview/PipeResourceMetrics.java | 38 +-
.../overview/PipeWALInsertNodeCacheMetrics.java | 86 ---
.../sink/PipeDataRegionConnectorMetrics.java | 34 ++
.../twostage/exchange/payload/CombineRequest.java | 1 -
.../payload/FetchCombineResultRequest.java | 1 -
.../protocol/thrift/IoTDBDataNodeReceiver.java | 16 +
.../pipe/resource/PipeDataNodeResourceManager.java | 25 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 33 +-
.../pipe/resource/memory/PipeMemoryWeightUtil.java | 8 +-
.../resource/tsfile/PipeCompactionManager.java | 68 +++
...Resource.java => PipeTsFilePublicResource.java} | 158 +-----
.../pipe/resource/tsfile/PipeTsFileResource.java | 253 +--------
.../resource/tsfile/PipeTsFileResourceManager.java | 329 ++++++------
.../db/pipe/resource/wal/PipeWALResource.java | 169 ------
.../pipe/resource/wal/PipeWALResourceManager.java | 142 -----
.../wal/hardlink/PipeWALHardlinkResource.java | 47 --
.../hardlink/PipeWALHardlinkResourceManager.java | 152 ------
.../wal/selfhost/PipeWALSelfHostResource.java | 41 --
.../selfhost/PipeWALSelfHostResourceManager.java | 38 --
.../impl/DataNodeInternalRPCServiceImpl.java | 93 +++-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 4 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 7 -
.../db/storageengine/dataregion/DataRegion.java | 4 +-
.../task/InsertionCrossSpaceCompactionTask.java | 2 +-
.../compaction/execute/utils/CompactionUtils.java | 7 +-
.../dataregion/flush/NotifyFlushMemTable.java | 7 -
.../dataregion/memtable/AbstractMemTable.java | 13 -
.../dataregion/memtable/IMemTable.java | 4 -
.../dataregion/memtable/TsFileProcessor.java | 26 +-
.../dataregion/tsfile/TsFileManager.java | 13 +
.../dataregion/tsfile/TsFileResource.java | 31 +-
.../dataregion/wal/buffer/WALBuffer.java | 6 -
.../dataregion/wal/buffer/WALEntry.java | 4 +-
.../wal/checkpoint/CheckpointManager.java | 70 +--
.../dataregion/wal/checkpoint/MemTableInfo.java | 19 +-
.../dataregion/wal/io/WALByteBufReader.java | 13 -
.../dataregion/wal/node/WALFakeNode.java | 4 +-
.../storageengine/dataregion/wal/node/WALNode.java | 96 +---
.../dataregion/wal/utils/WALEntryHandler.java | 233 --------
.../dataregion/wal/utils/WALEntryPosition.java | 242 ---------
.../dataregion/wal/utils/WALInsertNodeCache.java | 338 ------------
.../wal/utils/listener/WALFlushListener.java | 12 +-
.../db/storageengine/load/LoadTsFileManager.java | 1 +
...leStatementDataTypeConvertExecutionVisitor.java | 90 ++--
...eeStatementDataTypeConvertExecutionVisitor.java | 172 ++----
.../converter/LoadTsFileDataTypeConverter.java | 49 +-
.../execution/SubscriptionSubtaskExecutor.java | 9 +-
.../task/stage/SubscriptionTaskConnectorStage.java | 4 +-
.../connector/PipeDataNodeThriftRequestTest.java | 12 -
.../pipe/event/TsFileInsertionEventParserTest.java | 1 +
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 101 ++--
.../resource/PipeTsFileResourceManagerTest.java | 96 ++--
.../PipeWALHardlinkResourceManagerTest.java | 114 ----
.../TsFileResourceProgressIndexTest.java | 23 +-
.../wal/checkpoint/CheckpointManagerTest.java | 172 ------
.../dataregion/wal/node/WALEntryHandlerTest.java | 586 ---------------------
.../dataregion/wal/node/WALNodeTest.java | 17 -
.../wal/node/WalDeleteOutdatedNewTest.java | 65 ---
.../wal/utils/WALInsertNodeCacheTest.java | 190 -------
.../commons/concurrent/IoTDBThreadPoolFactory.java | 15 +
.../iotdb/commons/concurrent/ThreadName.java | 7 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 381 +++++---------
.../commons/consensus/index/ProgressIndex.java | 2 +-
.../consensus/index/impl/IoTProgressIndex.java | 27 +-
.../commons/pipe/agent/task/PipeTaskAgent.java | 58 +-
.../agent/task/execution/PipeSubtaskExecutor.java | 45 +-
.../task/progress/PipeEventCommitManager.java | 25 +-
.../agent/task/progress/PipeEventCommitter.java | 12 +-
.../task/subtask/PipeAbstractConnectorSubtask.java | 5 -
.../agent/task/subtask/PipeReportableSubtask.java | 21 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 137 ++---
.../iotdb/commons/pipe/config/PipeDescriptor.java | 124 ++---
.../config/constant/PipeConnectorConstant.java | 2 +-
.../thrift/request/PipeTransferFilePieceReq.java | 1 -
.../thrift/request/PipeTransferFileSealReqV1.java | 1 -
.../thrift/request/PipeTransferFileSealReqV2.java | 1 -
.../thrift/request/PipeTransferHandshakeV1Req.java | 1 -
.../thrift/request/PipeTransferHandshakeV2Req.java | 1 -
.../thrift/request/PipeTransferSliceReq.java | 1 -
.../pipe/connector/protocol/IoTDBConnector.java | 12 +
.../iotdb/commons/pipe/event/EnrichedEvent.java | 13 +
.../commons/pipe/resource/log/PipeLogStatus.java | 2 +-
.../iotdb/commons/service/metric/enums/Metric.java | 9 +-
168 files changed, 2529 insertions(+), 5342 deletions(-)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
index 43814100f27..78baab77218 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java
@@ -453,6 +453,12 @@ public class MppCommonConfig extends MppBaseConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ setProperty("pipe_enable_memory_checked", String.valueOf(isPipeEnableMemoryCheck));
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
setProperty("pipe_air_gap_receiver_enabled", String.valueOf(isPipeAirGapReceiverEnabled));
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
index 2ac6be4adad..6db1003f5b9 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java
@@ -466,6 +466,13 @@ public class MppSharedCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ dnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
+ cnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
dnConfig.setPipeAirGapReceiverEnabled(isPipeAirGapReceiverEnabled);
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
index 9c6dd3ac43c..55a772a60da 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java
@@ -328,6 +328,11 @@ public class RemoteCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
+ return this;
+ }
+
@Override
public CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled) {
return this;
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
index 872ac98d961..6f02ebfc770 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
@@ -146,6 +146,8 @@ public interface CommonConfig {
CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode);
+ CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck);
+
CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled);
CommonConfig setDriverTaskExecutionTimeSliceInMs(long driverTaskExecutionTimeSliceInMs);
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
index 50ff1c29814..d0f5eead050 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
@@ -40,6 +40,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -821,6 +822,7 @@ public class IoTDBLoadTsFileIT {
}
@Test
+ @Ignore("Load with conversion is currently banned")
public void testLoadWithConvertOnTypeMismatchForTreeModel() throws Exception {
List<Pair<MeasurementSchema, MeasurementSchema>> measurementSchemas =
@@ -943,6 +945,7 @@ public class IoTDBLoadTsFileIT {
}
@Test
+ @Ignore("Load with conversion is currently banned")
public void testLoadWithConvertOnTypeMismatchForTableModel() throws Exception {
final int lineCount = 10000;
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java
index fff0b8c6bf1..dd921f33bc7 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java
@@ -48,13 +48,15 @@ public abstract class AbstractPipeTableModelDualManualIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java
index 32075c4040c..5cf086b2db6 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java
@@ -72,13 +72,15 @@ public class IoTDBPipeExtractorIT extends AbstractPipeDualTreeModelAutoIT {
// Disable sender compaction for tsfile determination in loose range test
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
- .setEnableCrossSpaceCompaction(false);
+ .setEnableCrossSpaceCompaction(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java
index 53b5f27b0c4..9c755c1b0bf 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java
@@ -61,13 +61,15 @@ public class IoTDBPipeProcessorIT extends AbstractPipeDualTreeModelAutoIT {
.setAutoCreateSchemaEnabled(true)
.setTimestampPrecision("ms")
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
index b286cc61365..4358bc2a203 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
@@ -76,7 +76,8 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualTreeModelAutoIT {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
- .setDataReplicationFactor(dataRegionReplicationFactor);
+ .setDataReplicationFactor(dataRegionReplicationFactor)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
@@ -85,7 +86,8 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualTreeModelAutoIT {
.setSchemaRegionConsensusProtocolClass(schemaRegionConsensus)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
- .setDataReplicationFactor(dataRegionReplicationFactor);
+ .setDataReplicationFactor(dataRegionReplicationFactor)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java
index 34b42c35c14..ac70fc47349 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java
@@ -61,14 +61,16 @@ public class IoTDBPipeAutoConflictIT extends AbstractPipeDualTreeModelAutoIT {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java
index ff827d6332b..afde95f701b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java
@@ -76,7 +76,8 @@ public class IoTDBPipeClusterIT extends AbstractPipeDualTreeModelAutoIT {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
@@ -86,7 +87,8 @@ public class IoTDBPipeClusterIT extends AbstractPipeDualTreeModelAutoIT {
.setSchemaReplicationFactor(3)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java
index 37a542e6aa9..f978cbfe909 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java
@@ -68,14 +68,17 @@ public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualTreeModelAu
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
+
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setPipeAirGapReceiverEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java
index f9cfc7f9fe3..6034a9be97d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java
@@ -67,13 +67,15 @@ public class IoTDBPipeIdempotentIT extends AbstractPipeDualTreeModelAutoIT {
// of the tested idempotent sql.
.setDefaultSchemaRegionGroupNumPerDatabase(1)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java
index 0b929e4daf8..ea3d4252c44 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java
@@ -62,13 +62,15 @@ public class IoTDBPipeWithLoadIT extends AbstractPipeDualTreeModelAutoIT {
// Disable sender compaction to test mods
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
- .setEnableCrossSpaceCompaction(false);
+ .setEnableCrossSpaceCompaction(false)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java
index 581d363cdb5..bf0fe81482d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java
@@ -56,13 +56,15 @@ public abstract class AbstractPipeDualTreeModelManualIT {
.getCommonConfig()
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
- .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setIsPipeEnableMemoryCheck(false);
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java
index fd44c3d3383..27fb9f8c81b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java
@@ -45,6 +45,7 @@ import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -77,6 +78,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertTabletReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -95,6 +97,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedTabletReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -104,6 +107,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertRecordsReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -138,6 +142,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertRecordReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -178,6 +183,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedRecordReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -226,6 +232,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedRecordsReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -254,6 +261,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertStringRecordsOfOneDeviceReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -282,6 +290,7 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
}
@Test
+ @Ignore("The receiver conversion is currently banned, will ignore conflict")
public void insertAlignedStringRecordsOfOneDeviceReceiveByTsFile() {
prepareTypeConversionTest(
(ISession senderSession, ISession receiverSession, Tablet tablet) -> {
@@ -385,13 +394,12 @@ public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModel
String sql =
String.format(
"create pipe test%s"
- + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='%s','realtime.enable'='%s','history.enable'='%s')"
+ + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='%s','realtime.enable'='%s','history.enable'='true')"
+ " with processor ('processor'='do-nothing-processor')"
+ " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')",
diff,
isTSFile ? "file" : "forced-log",
!isTSFile,
- isTSFile,
receiverEnv.getIP(),
receiverEnv.getPort(),
isTSFile ? "tsfile" : "tablet");
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java
index e87bda0f361..e21024dfa14 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java
@@ -236,7 +236,7 @@ public class IoTDBPipeTypeConversionIT extends AbstractPipeDualTreeModelManualIT
String sql =
String.format(
"create pipe test"
- + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='forced-log','realtime.enable'='true','history.enable'='false')"
+ + " with source ('source'='iotdb-source','source.path'='root.test.**','realtime.mode'='forced-log')"
+ " with processor ('processor'='do-nothing-processor')"
+ " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='tablet')",
receiverEnv.getIP(), receiverEnv.getPort());
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
index 4c513576488..9a484426d76 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/AbstractPipeSingleIT.java
@@ -36,6 +36,7 @@ abstract class AbstractPipeSingleIT {
env.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true);
// 10 min, assert that the operations will not time out
env.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+ env.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
env.initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
index ba8123db596..c118166847c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/cluster/IoTDBSubscriptionRestartIT.java
@@ -82,7 +82,8 @@ public class IoTDBSubscriptionRestartIT extends AbstractSubscriptionIT {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(3)
- .setDataReplicationFactor(2);
+ .setDataReplicationFactor(2)
+ .setIsPipeEnableMemoryCheck(false);
EnvFactory.getEnv().initClusterEnvironment(3, 3);
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
index 594f9efe691..5e61607ff32 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/AbstractSubscriptionDualIT.java
@@ -58,6 +58,9 @@ public abstract class AbstractSubscriptionDualIT extends AbstractSubscriptionIT
// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
+
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java
index ece07a99161..6c7f726ddf4 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionConsumerGroupIT.java
@@ -118,8 +118,11 @@ public class IoTDBSubscriptionConsumerGroupIT extends AbstractSubscriptionDualIT
protected void setUpConfig() {
super.setUpConfig();
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+
// Enable air gap receiver
receiverEnv.getConfig().getCommonConfig().setPipeAirGapReceiverEnabled(true);
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTimePrecisionIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTimePrecisionIT.java
index f24b832153f..e77990e94db 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTimePrecisionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTimePrecisionIT.java
@@ -69,7 +69,9 @@ public class IoTDBSubscriptionTimePrecisionIT extends AbstractSubscriptionDualIT
// Set timestamp precision to nanosecond
senderEnv.getConfig().getCommonConfig().setTimestampPrecision("ns");
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
receiverEnv.getConfig().getCommonConfig().setTimestampPrecision("ns");
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Test
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTopicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTopicIT.java
index ccce4d9f0d2..0ccc6f4d927 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTopicIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/dual/treemodel/IoTDBSubscriptionTopicIT.java
@@ -92,6 +92,9 @@ public class IoTDBSubscriptionTopicIT extends AbstractSubscriptionDualIT {
.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(30);
senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerInitialSyncDelayMinutes(1);
senderEnv.getConfig().getCommonConfig().setPipeMetaSyncerSyncIntervalMinutes(1);
+ senderEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+
+ receiverEnv.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Test
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
index 3f415051928..7b564023498 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/AbstractSubscriptionLocalIT.java
@@ -33,7 +33,12 @@ public abstract class AbstractSubscriptionLocalIT extends AbstractSubscriptionIT
super.setUp();
// enable subscription
- EnvFactory.getEnv().getConfig().getCommonConfig().setSubscriptionEnabled(true);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSubscriptionEnabled(true)
+ .setIsPipeEnableMemoryCheck(false);
+
EnvFactory.getEnv().initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
index 8fa1e6854fa..b2cfede0945 100644
--- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/AbstractSubscriptionTripleIT.java
@@ -71,11 +71,14 @@ public abstract class AbstractSubscriptionTripleIT extends AbstractSubscriptionI
receiver2.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
// reduce tsfile batch memory usage
- sender.getConfig().getCommonConfig().setSubscriptionPrefetchTsFileBatchMaxDelayInMs(500);
sender
.getConfig()
.getCommonConfig()
+ .setIsPipeEnableMemoryCheck(false)
+ .setSubscriptionPrefetchTsFileBatchMaxDelayInMs(500)
.setSubscriptionPrefetchTsFileBatchMaxSizeInBytes(32 * 1024);
+ receiver1.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
+ receiver2.getConfig().getCommonConfig().setIsPipeEnableMemoryCheck(false);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
index 334967b51d8..da8ffdc6823 100644
--- a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportTsFileTestIT.java
@@ -53,7 +53,11 @@ public class ExportTsFileTestIT extends AbstractScriptIT {
@BeforeClass
public static void setUp() throws Exception {
// enable subscription
- EnvFactory.getEnv().getConfig().getCommonConfig().setSubscriptionEnabled(true);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSubscriptionEnabled(true)
+ .setIsPipeEnableMemoryCheck(false);
EnvFactory.getEnv().initClusterEnvironment();
ip = EnvFactory.getEnv().getIP();
diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
index 3d041867242..4c7fffcfba5 100644
--- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
+++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/event/dml/insertion/TsFileInsertionEvent.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.pipe.api.event.dml.insertion;
import org.apache.iotdb.pipe.api.event.Event;
+import java.io.File;
+
/**
* {@link TsFileInsertionEvent} is used to define the event of writing TsFile. Event data stores in
* disks, which is compressed and encoded, and requires IO cost for computational processing.
@@ -34,4 +36,12 @@ public interface TsFileInsertionEvent extends Event, AutoCloseable {
* @return {@code Iterable<TabletInsertionEvent>} the list of {@link TabletInsertionEvent}
*/
Iterable<TabletInsertionEvent> toTabletInsertionEvents();
+
+ /**
+ * Get the file that stores the data of this {@link TsFileInsertionEvent}. The file is compressed
+ * and encoded, and requires IO cost for computational processing.
+ *
+ * @return the file that stores the data of this {@link TsFileInsertionEvent}
+ */
+ File getTsFile();
}
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 7249df56230..d153e425dba 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -285,6 +285,7 @@ public enum TSStatusCode {
PIPE_RECEIVER_USER_CONFLICT_EXCEPTION(1810),
PIPE_CONFIG_RECEIVER_HANDSHAKE_NEEDED(1811),
PIPE_TRANSFER_SLICE_OUT_OF_ORDER(1812),
+ PIPE_PUSH_META_TIMEOUT(1813),
// Subscription
SUBSCRIPTION_VERSION_ERROR(1900),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java
index 9aea3edfa38..1ac64eb246f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeSubtaskExecutor.java
@@ -28,7 +28,7 @@ public class PipeConfigNodeSubtaskExecutor extends PipeSubtaskExecutor {
private static final int THREAD_NUM = 1;
private PipeConfigNodeSubtaskExecutor() {
- super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL, true);
+ super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL.getName(), true);
}
/**
@@ -36,7 +36,7 @@ public class PipeConfigNodeSubtaskExecutor extends PipeSubtaskExecutor {
*/
@TestOnly
public PipeConfigNodeSubtaskExecutor(final Object ignored) {
- super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL, true);
+ super(THREAD_NUM, ThreadName.PIPE_CONFIGNODE_EXECUTOR_POOL.getName(), true);
}
private static class PipeSchemaSubtaskExecutorHolder {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
index ae20141c81f..b79fd8da94f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java
@@ -46,11 +46,13 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
@@ -248,4 +250,10 @@ public class PipeConfigNodeTaskAgent extends PipeTaskAgent {
resp.setPipeRemainingEventCountList(pipeRemainingEventCountList);
resp.setPipeRemainingTimeList(pipeRemainingTimeList);
}
+
+ @Override
+ public void runPipeTasks(
+ final Collection<PipeTask> pipeTasks, final Consumer<PipeTask> runSingle) {
+ pipeTasks.forEach(runSingle);
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
index a9654fd3080..7a3a899e1bf 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
@@ -426,12 +426,20 @@ public abstract class AbstractOperatePipeProcedureV2
* @return Error messages for the given pipe after pushing pipe meta
*/
public static String parsePushPipeMetaExceptionForPipe(
- String pipeName, Map<Integer, TPushPipeMetaResp> respMap) {
+ final String pipeName, final Map<Integer, TPushPipeMetaResp> respMap) {
final StringBuilder exceptionMessageBuilder = new StringBuilder();
- for (Map.Entry<Integer, TPushPipeMetaResp> respEntry : respMap.entrySet()) {
- int dataNodeId = respEntry.getKey();
- TPushPipeMetaResp resp = respEntry.getValue();
+ for (final Map.Entry<Integer, TPushPipeMetaResp> respEntry : respMap.entrySet()) {
+ final int dataNodeId = respEntry.getKey();
+ final TPushPipeMetaResp resp = respEntry.getValue();
+
+ if (resp.getStatus().getCode() == TSStatusCode.PIPE_PUSH_META_TIMEOUT.getStatusCode()) {
+ exceptionMessageBuilder.append(
+ String.format(
+ "DataNodeId: %s, Message: Timeout to wait for lock while processing pushPipeMeta on dataNodes.",
+ dataNodeId));
+ continue;
+ }
if (resp.getStatus().getCode() == TSStatusCode.PIPE_PUSH_META_ERROR.getStatusCode()) {
if (!resp.isSetExceptionMessages()) {
@@ -442,7 +450,7 @@ public abstract class AbstractOperatePipeProcedureV2
continue;
}
- AtomicBoolean hasException = new AtomicBoolean(false);
+ final AtomicBoolean hasException = new AtomicBoolean(false);
resp.getExceptionMessages()
.forEach(
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
index 1ee65cb137e..a18ba7a4bb4 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/connector/PipeConfigNodeThriftRequestTest.java
@@ -44,7 +44,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getTimestampPrecision(), deserializeReq.getTimestampPrecision());
}
@@ -57,7 +56,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
}
@Test
@@ -72,7 +70,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileName(), deserializeReq.getFileName());
Assert.assertEquals(req.getStartWritingOffset(), deserializeReq.getStartWritingOffset());
@@ -105,7 +102,6 @@ public class PipeConfigNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileNames(), deserializeReq.getFileNames());
Assert.assertEquals(req.getFileLengths(), deserializeReq.getFileLengths());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 8137959a391..b751c150585 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -280,6 +280,8 @@ public class IoTDBConfig {
private String extPipeDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_PIPE_FOLDER_NAME;
+ private int pipeTaskThreadCount = 5;
+
/** External lib directory for MQTT, stores user-uploaded JAR files */
private String mqttDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.MQTT_FOLDER_NAME;
@@ -968,6 +970,7 @@ public class IoTDBConfig {
private long generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch = 600L;
private int generalRegionAttributeSecurityServiceFailureTimesToFetch = 20;
private long detailContainerMinDegradeMemoryInBytes = 1024 * 1024L;
+ private int schemaThreadCount = 5;
private String readConsistencyLevel = "strong";
@@ -1086,7 +1089,6 @@ public class IoTDBConfig {
private int loadTsFileTabletConversionThreadCount = 5;
private int loadTsFileMaxDeviceCountToUseDeviceTimeIndex = 10000;
-
private long loadChunkMetadataMemorySizeInBytes = 33554432; // 32MB
private long loadMemoryAllocateRetryIntervalMs = 1000L;
@@ -3118,6 +3120,14 @@ public class IoTDBConfig {
this.extPipeDir = extPipeDir;
}
+ public int getPipeTaskThreadCount() {
+ return pipeTaskThreadCount;
+ }
+
+ public void setPipeTaskThreadCount(int pipeTaskThreadCount) {
+ this.pipeTaskThreadCount = pipeTaskThreadCount;
+ }
+
public void setPartitionCacheSize(int partitionCacheSize) {
this.partitionCacheSize = partitionCacheSize;
}
@@ -3279,6 +3289,14 @@ public class IoTDBConfig {
this.detailContainerMinDegradeMemoryInBytes = detailContainerMinDegradeMemoryInBytes;
}
+ public int getSchemaThreadCount() {
+ return schemaThreadCount;
+ }
+
+ public void setSchemaThreadCount(int schemaThreadCount) {
+ this.schemaThreadCount = schemaThreadCount;
+ }
+
public String getReadConsistencyLevel() {
return readConsistencyLevel;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index a4d199f662d..34501ffff69 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -895,7 +895,11 @@ public class IoTDBDescriptor {
conf.setIntoOperationExecutionThreadCount(2);
}
- conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir()));
+ conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir()).trim());
+ conf.setPipeTaskThreadCount(
+ Integer.parseInt(
+ properties.getProperty(
+ "pipe_task_thread_count", Integer.toString(conf.getPipeTaskThreadCount()).trim())));
// At the same time, set TSFileConfig
List<FSType> fsTypes = new ArrayList<>();
@@ -1075,6 +1079,10 @@ public class IoTDBDescriptor {
properties.getProperty(
"detail_container_min_degrade_memory_in_bytes",
String.valueOf(conf.getDetailContainerMinDegradeMemoryInBytes()))));
+ conf.setSchemaThreadCount(
+ Integer.parseInt(
+ properties.getProperty(
+ "schema_thread_count", Integer.toString(conf.getSchemaThreadCount()))));
loadIoTConsensusProps(properties);
loadIoTConsensusV2Props(properties);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
index 96e2e4e7299..089d4e2fda7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeAgentLauncher.java
@@ -36,7 +36,6 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
import org.apache.iotdb.db.service.ResourcesInformationHolder;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
import org.apache.iotdb.pipe.api.exception.PipeException;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -161,7 +160,6 @@ class PipeAgentLauncher {
try (final ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
final TGetAllPipeInfoResp getAllPipeInfoResp = configNodeClient.getAllPipeInfo();
- WALInsertNodeCache.init();
PipeTabletEventBatch.init();
if (getAllPipeInfoResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
throw new StartupException("Failed to get pipe task meta from config node.");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
index ba6ab4c66cd..9c4b0419794 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
@@ -35,7 +35,6 @@ import org.apache.iotdb.commons.service.ServiceType;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningQueue;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
@@ -86,15 +85,6 @@ public class PipeDataNodeRuntimeAgent implements IService {
PipeConfig.getInstance().printAllConfigs();
PipeAgentLauncher.launchPipeTaskAgent();
- registerPeriodicalJob(
- "PipeTaskAgent#restartAllStuckPipes",
- PipeDataNodeAgent.task()::restartAllStuckPipes,
- PipeConfig.getInstance().getPipeStuckRestartIntervalSeconds());
- registerPeriodicalJob(
- "PipeTaskAgent#flushDataRegionIfNeeded",
- PipeTerminateEvent::flushDataRegionIfNeeded,
- PipeConfig.getInstance().getPipeFlushAfterLastTerminateSeconds());
-
pipePeriodicalJobExecutor.start();
if (PipeConfig.getInstance().getPipeEventReferenceTrackingEnabled()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
index ab78200f688..7606cbc4ea6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
@@ -21,6 +21,10 @@ package org.apache.iotdb.db.pipe.agent.task;
import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.consensus.SchemaRegionId;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
@@ -37,8 +41,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeType;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant;
import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
-import org.apache.iotdb.commons.service.metric.MetricService;
-import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -48,13 +51,12 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.agent.task.builder.PipeDataNodeBuilder;
import org.apache.iotdb.db.pipe.agent.task.builder.PipeDataNodeTaskBuilder;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
-import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.extractor.schemaregion.SchemaRegionListeningFilter;
import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
-import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
@@ -62,10 +64,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.SystemMetric;
import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp;
import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaRespExceptionMessage;
@@ -83,6 +82,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -91,19 +91,42 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_START_TIME_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATH_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATTERN_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_START_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_END_TIME_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_START_TIME_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_SNAPSHOT_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_PATH_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_PATTERN_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY;
+import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_START_TIME_KEY;
public class PipeDataNodeTaskAgent extends PipeTaskAgent {
@@ -111,10 +134,17 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
protected static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final AtomicLong LAST_FORCED_RESTART_TIME =
- new AtomicLong(System.currentTimeMillis());
- private static final Map<String, AtomicLong> PIPE_NAME_TO_LAST_RESTART_TIME_MAP =
- new ConcurrentHashMap<>();
+ private final ExecutorService pipeExecutor =
+ new WrappedThreadPoolExecutor(
+ 0,
+ IoTDBDescriptor.getInstance().getConfig().getPipeTaskThreadCount(),
+ 0L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount()),
+ new IoTThreadFactory(ThreadName.PIPE_PARALLEL_EXECUTION_POOL.getName()),
+ ThreadName.PIPE_PARALLEL_EXECUTION_POOL.getName(),
+ new ThreadPoolExecutor.CallerRunsPolicy());
////////////////////////// Pipe Task Management Entry //////////////////////////
@@ -131,54 +161,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
: new PipeDataNodeBuilder(pipeMetaFromConfigNode).buildTasksWithInternalSource();
}
- ////////////////////////// Manage by Pipe Name //////////////////////////
-
- @Override
- protected void startPipe(final String pipeName, final long creationTime) {
- final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName);
- final PipeStatus status = existedPipeMeta.getRuntimeMeta().getStatus().get();
- if (PipeStatus.STOPPED.equals(status) || status == null) {
- restartPipeToReloadResourceIfNeeded(existedPipeMeta);
- }
-
- super.startPipe(pipeName, creationTime);
- }
-
- private void restartPipeToReloadResourceIfNeeded(final PipeMeta pipeMeta) {
- if (System.currentTimeMillis() - pipeMeta.getStaticMeta().getCreationTime()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()) {
- return;
- }
-
- final AtomicLong lastRestartTime =
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP.get(pipeMeta.getStaticMeta().getPipeName());
- if (lastRestartTime != null
- && System.currentTimeMillis() - lastRestartTime.get()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()) {
- LOGGER.info(
- "Skipping reload resource for stopped pipe {} before starting it because reloading resource is too frequent.",
- pipeMeta.getStaticMeta().getPipeName());
- return;
- }
-
- if (PIPE_NAME_TO_LAST_RESTART_TIME_MAP.isEmpty()) {
- LOGGER.info(
- "Flushing storage engine before restarting pipe {}.",
- pipeMeta.getStaticMeta().getPipeName());
- final long currentTime = System.currentTimeMillis();
- StorageEngine.getInstance().syncCloseAllProcessor();
- WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes();
- LOGGER.info(
- "Finished flushing storage engine, time cost: {} ms.",
- System.currentTimeMillis() - currentTime);
- }
-
- restartStuckPipe(pipeMeta);
- LOGGER.info(
- "Reloaded resource for stopped pipe {} before starting it.",
- pipeMeta.getStaticMeta().getPipeName());
- }
-
///////////////////////// Manage by regionGroupId /////////////////////////
@Override
@@ -399,9 +381,8 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
///////////////////////// Heartbeat /////////////////////////
public void collectPipeMetaList(final TDataNodeHeartbeatResp resp) throws TException {
- // Try the lock instead of directly acquire it to prevent the block of the cluster heartbeat
- // 10s is the half of the HEARTBEAT_TIMEOUT_TIME defined in class BaseNodeCache in ConfigNode
- if (!tryReadLockWithTimeOut(10)) {
+ if (!tryReadLockWithTimeOut(
+ CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS() * 2L / 3)) {
return;
}
try {
@@ -564,226 +545,6 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
PipeInsertionDataNodeListener.getInstance().listenToHeartbeat(true);
}
- ///////////////////////// Restart Logic /////////////////////////
-
- public void restartAllStuckPipes() {
- final List<String> removedPipeName = removeOutdatedPipeInfoFromLastRestartTimeMap();
- if (!removedPipeName.isEmpty()) {
- final long currentTime = System.currentTimeMillis();
- LOGGER.info(
- "Pipes {} now can dynamically adjust their extraction strategies. "
- + "Start to flush storage engine to trigger the adjustment.",
- removedPipeName);
- StorageEngine.getInstance().syncCloseAllProcessor();
- WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes();
- LOGGER.info(
- "Finished flushing storage engine, time cost: {} ms.",
- System.currentTimeMillis() - currentTime);
- LOGGER.info("Skipping restarting pipes this round because of the dynamic flushing.");
- return;
- }
-
- if (!tryWriteLockWithTimeOut(5)) {
- return;
- }
-
- final Set<PipeMeta> stuckPipes;
- try {
- stuckPipes = findAllStuckPipes();
- } finally {
- releaseWriteLock();
- }
-
- // If the pipe has been restarted recently, skip it.
- stuckPipes.removeIf(
- pipeMeta -> {
- final AtomicLong lastRestartTime =
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP.get(pipeMeta.getStaticMeta().getPipeName());
- return lastRestartTime != null
- && System.currentTimeMillis() - lastRestartTime.get()
- < PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs();
- });
-
- // Restart all stuck pipes.
- // Note that parallelStream cannot be used here. The method PipeTaskAgent#dropPipe also uses
- // parallelStream. If parallelStream is used here, the subtasks generated inside the dropPipe
- // may not be scheduled by the worker thread of ForkJoinPool because of less available threads,
- // and the parent task will wait for the completion of the subtasks in ForkJoinPool forever,
- // causing the deadlock.
- stuckPipes.forEach(this::restartStuckPipe);
- }
-
- private List<String> removeOutdatedPipeInfoFromLastRestartTimeMap() {
- final List<String> removedPipeName = new ArrayList<>();
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP
- .entrySet()
- .removeIf(
- entry -> {
- final AtomicLong lastRestartTime = entry.getValue();
- final boolean shouldRemove =
- lastRestartTime == null
- || PipeConfig.getInstance().getPipeStuckRestartMinIntervalMs()
- <= System.currentTimeMillis() - lastRestartTime.get();
- if (shouldRemove) {
- removedPipeName.add(entry.getKey());
- }
- return shouldRemove;
- });
- return removedPipeName;
- }
-
- private Set<PipeMeta> findAllStuckPipes() {
- final Set<PipeMeta> stuckPipes = new HashSet<>();
-
- if (System.currentTimeMillis() - LAST_FORCED_RESTART_TIME.get()
- > PipeConfig.getInstance().getPipeSubtaskExecutorForcedRestartIntervalMs()) {
- LAST_FORCED_RESTART_TIME.set(System.currentTimeMillis());
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- stuckPipes.add(pipeMeta);
- }
- if (!stuckPipes.isEmpty()) {
- LOGGER.warn(
- "All {} pipe(s) will be restarted because of forced restart policy.",
- stuckPipes.size());
- }
- return stuckPipes;
- }
-
- final long totalLinkedButDeletedTsFileResourceRamSize =
- PipeDataNodeResourceManager.tsfile().getTotalLinkedButDeletedTsFileResourceRamSize();
- final long totalInsertNodeFloatingMemoryUsageInBytes = getAllFloatingMemoryUsageInByte();
- final long totalFloatingMemorySizeInBytes =
- PipeDataNodeResourceManager.memory().getTotalFloatingMemorySizeInBytes();
- if (totalInsertNodeFloatingMemoryUsageInBytes + totalLinkedButDeletedTsFileResourceRamSize
- >= totalFloatingMemorySizeInBytes) {
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- stuckPipes.add(pipeMeta);
- }
- if (!stuckPipes.isEmpty()) {
- LOGGER.warn(
- "All {} pipe(s) will be restarted because linked but deleted tsFiles' resource size {} and all insertNode's size {} exceeds limit {}.",
- stuckPipes.size(),
- totalLinkedButDeletedTsFileResourceRamSize,
- totalInsertNodeFloatingMemoryUsageInBytes,
- totalFloatingMemorySizeInBytes);
- }
- return stuckPipes;
- }
-
- final Map<String, IoTDBDataRegionExtractor> taskId2ExtractorMap =
- PipeDataRegionExtractorMetrics.getInstance().getExtractorMap();
- for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) {
- final String pipeName = pipeMeta.getStaticMeta().getPipeName();
- final List<IoTDBDataRegionExtractor> extractors =
- taskId2ExtractorMap.values().stream()
- .filter(e -> e.getPipeName().equals(pipeName) && e.shouldExtractInsertion())
- .collect(Collectors.toList());
-
- if (extractors.isEmpty()) {
- continue;
- }
-
- // Extractors of this pipe might not pin too much MemTables,
- // still need to check if linked-and-deleted TsFile count exceeds limit.
- // Typically, if deleted tsFiles are too abundant all pipes may need to restart.
- if ((CONFIG.isEnableSeqSpaceCompaction()
- || CONFIG.isEnableUnseqSpaceCompaction()
- || CONFIG.isEnableCrossSpaceCompaction())
- && mayDeletedTsFileSizeReachDangerousThreshold()) {
- LOGGER.warn(
- "Pipe {} needs to restart because too many TsFiles are out-of-date.",
- pipeMeta.getStaticMeta());
- stuckPipes.add(pipeMeta);
- continue;
- }
-
- // Try to restart the stream mode pipes for releasing memTables.
- if (extractors.get(0).isStreamMode()) {
- if (extractors.stream().anyMatch(IoTDBDataRegionExtractor::hasConsumedAllHistoricalTsFiles)
- && (mayMemTablePinnedCountReachDangerousThreshold()
- || mayWalSizeReachThrottleThreshold())) {
- // Extractors of this pipe may be stuck and is pinning too many MemTables.
- LOGGER.warn(
- "Pipe {} needs to restart because too many memTables are pinned or the WAL size is too large. mayMemTablePinnedCountReachDangerousThreshold: {}, mayWalSizeReachThrottleThreshold: {}",
- pipeMeta.getStaticMeta(),
- mayMemTablePinnedCountReachDangerousThreshold(),
- mayWalSizeReachThrottleThreshold());
- stuckPipes.add(pipeMeta);
- }
- }
- }
-
- return stuckPipes;
- }
-
- private boolean mayDeletedTsFileSizeReachDangerousThreshold() {
- try {
- final long linkedButDeletedTsFileSize =
- PipeDataNodeResourceManager.tsfile().getTotalLinkedButDeletedTsfileSize();
- final double totalDisk =
- MetricService.getInstance()
- .getAutoGauge(
- SystemMetric.SYS_DISK_TOTAL_SPACE.toString(),
- MetricLevel.CORE,
- Tag.NAME.toString(),
- // This "system" should stay the same with the one in
- // DataNodeInternalRPCServiceImpl.
- "system")
- .getValue();
- return linkedButDeletedTsFileSize > 0
- && totalDisk > 0
- && linkedButDeletedTsFileSize
- > PipeConfig.getInstance().getPipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage()
- * totalDisk;
- } catch (final Exception e) {
- LOGGER.warn("Failed to judge if deleted TsFile size reaches dangerous threshold.", e);
- return false;
- }
- }
-
- private boolean mayMemTablePinnedCountReachDangerousThreshold() {
- return PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount() != Integer.MAX_VALUE
- && PipeDataNodeResourceManager.wal().getPinnedWalCount()
- >= 5
- * PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber();
- }
-
- private boolean mayWalSizeReachThrottleThreshold() {
- return 3 * WALManager.getInstance().getTotalDiskUsage() > 2 * CONFIG.getThrottleThreshold();
- }
-
- private void restartStuckPipe(final PipeMeta pipeMeta) {
- LOGGER.warn(
- "Pipe {} will be restarted because it is stuck or has encountered issues such as data backlog or being stopped for too long.",
- pipeMeta.getStaticMeta());
- acquireWriteLock();
- try {
- final long startTime = System.currentTimeMillis();
- final PipeMeta originalPipeMeta = pipeMeta.deepCopy4TaskAgent();
- handleDropPipe(pipeMeta.getStaticMeta().getPipeName());
-
- final long restartTime = System.currentTimeMillis();
- PIPE_NAME_TO_LAST_RESTART_TIME_MAP
- .computeIfAbsent(pipeMeta.getStaticMeta().getPipeName(), k -> new AtomicLong(restartTime))
- .set(restartTime);
- handleSinglePipeMetaChanges(originalPipeMeta);
-
- LOGGER.warn(
- "Pipe {} was restarted because of stuck or data backlog, time cost: {} ms.",
- originalPipeMeta.getStaticMeta(),
- System.currentTimeMillis() - startTime);
- } catch (final Exception e) {
- LOGGER.warn("Failed to restart stuck pipe {}.", pipeMeta.getStaticMeta(), e);
- } finally {
- releaseWriteLock();
- }
- }
-
- public boolean isPipeTaskCurrentlyRestarted(final String pipeName) {
- return PIPE_NAME_TO_LAST_RESTART_TIME_MAP.containsKey(pipeName);
- }
-
///////////////////////// Terminate Logic /////////////////////////
public void markCompleted(final String pipeName, final int regionId) {
@@ -845,6 +606,24 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
return isSnapshotMode;
}
+ @Override
+ public void runPipeTasks(
+ final Collection<PipeTask> pipeTasks, final Consumer<PipeTask> runSingle) {
+ final Set<Future<?>> pipeFuture = new HashSet<>();
+
+ pipeTasks.forEach(
+ pipeTask -> pipeFuture.add(pipeExecutor.submit(() -> runSingle.accept(pipeTask))));
+
+ for (final Future<?> future : pipeFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing pipe task: ", e);
+ throw new PipeException(e.toString());
+ }
+ }
+ }
+
///////////////////////// Shutdown Logic /////////////////////////
public void persistAllProgressIndex() {
@@ -908,4 +687,204 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
releaseReadLock();
}
}
+
+ @Override
+ protected void calculateMemoryUsage(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+ if (!PipeConfig.getInstance().isPipeEnableMemoryCheck()) {
+ return;
+ }
+
+ calculateInsertNodeQueueMemory(extractorParameters, processorParameters, connectorParameters);
+
+ long needMemory = 0;
+
+ needMemory +=
+ calculateTsFileParserMemory(extractorParameters, processorParameters, connectorParameters);
+ needMemory +=
+ calculateSinkBatchMemory(extractorParameters, processorParameters, connectorParameters);
+ needMemory +=
+ calculateSendTsFileReadBufferMemory(
+ extractorParameters, processorParameters, connectorParameters);
+
+ PipeMemoryManager pipeMemoryManager = PipeDataNodeResourceManager.memory();
+ final long freeMemorySizeInBytes = pipeMemoryManager.getFreeMemorySizeInBytes();
+ final long reservedMemorySizeInBytes =
+ (long)
+ (PipeDataNodeResourceManager.memory().getTotalMemorySizeInBytes()
+ * PipeConfig.getInstance().getReservedMemoryPercentage());
+ if (freeMemorySizeInBytes < needMemory + reservedMemorySizeInBytes) {
+ final String message =
+ String.format(
+ "Not enough memory for pipe. Need memory: %d bytes, free memory: %d bytes, reserved memory: %d bytes, total memory: %d bytes",
+ needMemory,
+ freeMemorySizeInBytes,
+ freeMemorySizeInBytes,
+ PipeDataNodeResourceManager.memory().getTotalMemorySizeInBytes());
+ LOGGER.warn(message);
+ throw new PipeException(message);
+ }
+ }
+
+ private void calculateInsertNodeQueueMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // Realtime extractor is enabled by default, so we only need to check the source realtime
+ if (!extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
+ EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)) {
+ return;
+ }
+
+ // If the realtime mode is batch or file, we do not need to allocate memory
+ final String realtimeMode =
+ extractorParameters.getStringByKeys(
+ PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_KEY,
+ PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY);
+ if (PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE.equals(realtimeMode)
+ || PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FILE_VALUE.equals(realtimeMode)) {
+ return;
+ }
+
+ final long allocatedMemorySizeInBytes = this.getAllFloatingMemoryUsageInByte();
+ final long remainingMemory =
+ PipeDataNodeResourceManager.memory().getTotalFloatingMemorySizeInBytes()
+ - allocatedMemorySizeInBytes;
+ if (remainingMemory < PipeConfig.getInstance().PipeInsertNodeQueueMemory()) {
+ final String message =
+ String.format(
+ "Not enough memory for pipe. Need Floating memory: %d bytes, free Floating memory: %d bytes",
+ PipeConfig.getInstance().PipeInsertNodeQueueMemory(), remainingMemory);
+ LOGGER.warn(message);
+ throw new PipeException(message);
+ }
+ }
+
+ private long calculateTsFileParserMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // If the extractor is not history, we do not need to allocate memory
+ boolean isExtractorHistory =
+ extractorParameters.getBooleanOrDefault(
+ SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE);
+
+ // If the extractor is history, and has start/end time, we need to allocate memory
+ boolean isTSFileParser =
+ isExtractorHistory
+ && extractorParameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_START_TIME_KEY, SOURCE_HISTORY_START_TIME_KEY);
+
+ isTSFileParser =
+ isTSFileParser
+ || (isExtractorHistory
+ && extractorParameters.hasAnyAttributes(
+ EXTRACTOR_HISTORY_END_TIME_KEY, SOURCE_HISTORY_END_TIME_KEY));
+
+ // if the extractor has start/end time, we need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(
+ SOURCE_START_TIME_KEY, EXTRACTOR_START_TIME_KEY);
+
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(SOURCE_END_TIME_KEY, EXTRACTOR_END_TIME_KEY);
+
+ // If the extractor has pattern or path, we need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || extractorParameters.hasAnyAttributes(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY);
+
+ isTSFileParser =
+ isTSFileParser || extractorParameters.hasAnyAttributes(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY);
+
+ // If the extractor is not hybrid, we do need to allocate memory
+ isTSFileParser =
+ isTSFileParser
+ || !PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals(
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY,
+ PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE));
+
+ if (!isTSFileParser) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getTsFileParserMemory();
+ }
+
+ private long calculateSinkBatchMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+
+ // If the connector format is tsfile , we need to use batch
+ boolean needUseBatch =
+ PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE.equals(
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY,
+ PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE));
+
+ if (needUseBatch) {
+ return PipeConfig.getInstance().getSinkBatchMemoryTsFile();
+ }
+
+ // If the connector is batch mode, we need to use batch
+ needUseBatch =
+ connectorParameters.getBooleanOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY,
+ PipeConnectorConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY),
+ PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_DEFAULT_VALUE);
+
+ if (!needUseBatch) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getSinkBatchMemoryInsertNode();
+ }
+
+ private long calculateSendTsFileReadBufferMemory(
+ final PipeParameters extractorParameters,
+ final PipeParameters processorParameters,
+ final PipeParameters connectorParameters) {
+ // If the extractor is history enable, we need to transfer tsfile
+ boolean needTransferTsFile =
+ extractorParameters.getBooleanOrDefault(
+ SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || extractorParameters.getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE);
+
+ String format =
+ connectorParameters.getStringOrDefault(
+ Arrays.asList(
+ PipeConnectorConstant.CONNECTOR_FORMAT_KEY, PipeConnectorConstant.SINK_FORMAT_KEY),
+ PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE);
+
+ // If the connector format is tsfile and hybrid, we need to transfer tsfile
+ needTransferTsFile =
+ needTransferTsFile
+ || PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE.equals(format)
+ || PipeConnectorConstant.CONNECTOR_FORMAT_TS_FILE_VALUE.equals(format);
+
+ if (!needTransferTsFile) {
+ return 0;
+ }
+
+ return PipeConfig.getInstance().getSendTsFileReadBuffer();
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
index 0df7f1c133b..496c1d61005 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/builder/PipeDataNodeTaskBuilder.java
@@ -29,7 +29,6 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant;
import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTask;
-import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeProcessorSubtaskExecutor;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeSubtaskExecutorManager;
import org.apache.iotdb.db.pipe.agent.task.stage.PipeTaskConnectorStage;
@@ -44,7 +43,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
-import java.util.EnumMap;
import java.util.HashMap;
import java.util.Map;
@@ -72,19 +70,8 @@ public class PipeDataNodeTaskBuilder {
private final int regionId;
private final PipeTaskMeta pipeTaskMeta;
- private static final PipeProcessorSubtaskExecutor PROCESSOR_EXECUTOR;
- private static final Map<PipeType, PipeConnectorSubtaskExecutor> CONNECTOR_EXECUTOR_MAP;
-
- static {
- PROCESSOR_EXECUTOR = PipeSubtaskExecutorManager.getInstance().getProcessorExecutor();
- CONNECTOR_EXECUTOR_MAP = new EnumMap<>(PipeType.class);
- CONNECTOR_EXECUTOR_MAP.put(
- PipeType.USER, PipeSubtaskExecutorManager.getInstance().getConnectorExecutor());
- CONNECTOR_EXECUTOR_MAP.put(
- PipeType.SUBSCRIPTION, PipeSubtaskExecutorManager.getInstance().getSubscriptionExecutor());
- CONNECTOR_EXECUTOR_MAP.put(
- PipeType.CONSENSUS, PipeSubtaskExecutorManager.getInstance().getConsensusExecutor());
- }
+ private static final PipeProcessorSubtaskExecutor PROCESSOR_EXECUTOR =
+ PipeSubtaskExecutorManager.getInstance().getProcessorExecutor();
protected final Map<String, String> systemParameters = new HashMap<>();
@@ -126,7 +113,7 @@ public class PipeDataNodeTaskBuilder {
pipeStaticMeta.getCreationTime(),
connectorParameters,
regionId,
- CONNECTOR_EXECUTOR_MAP.get(pipeType));
+ PipeSubtaskExecutorManager.getInstance().getSubscriptionExecutor());
} else { // user pipe or consensus pipe
connectorStage =
new PipeTaskConnectorStage(
@@ -134,7 +121,9 @@ public class PipeDataNodeTaskBuilder {
pipeStaticMeta.getCreationTime(),
connectorParameters,
regionId,
- CONNECTOR_EXECUTOR_MAP.get(pipeType));
+ pipeType.equals(PipeType.USER)
+ ? PipeSubtaskExecutorManager.getInstance().getConnectorExecutorSupplier()
+ : PipeSubtaskExecutorManager.getInstance().getConsensusExecutorSupplier());
}
// The processor connects the extractor and connector.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java
index fcb4888dbde..0850c81a59d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeConnectorSubtaskExecutor.java
@@ -23,16 +23,20 @@ import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.pipe.agent.task.execution.PipeSubtaskExecutor;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import java.util.concurrent.atomic.AtomicInteger;
+
public class PipeConnectorSubtaskExecutor extends PipeSubtaskExecutor {
+ private static final AtomicInteger id = new AtomicInteger(0);
public PipeConnectorSubtaskExecutor() {
super(
PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(),
- ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL,
+ ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL.getName() + "-" + id.get(),
+ ThreadName.PIPE_SUBTASK_CALLBACK_EXECUTOR_POOL.getName() + "-" + id.getAndIncrement(),
true);
}
- public PipeConnectorSubtaskExecutor(final int corePoolSize, final ThreadName threadName) {
+ public PipeConnectorSubtaskExecutor(final int corePoolSize, final String threadName) {
super(corePoolSize, threadName, true);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java
index f25d28b0c2c..9eddd27f22e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeProcessorSubtaskExecutor.java
@@ -28,7 +28,7 @@ public class PipeProcessorSubtaskExecutor extends PipeSubtaskExecutor {
public PipeProcessorSubtaskExecutor() {
super(
PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(),
- ThreadName.PIPE_PROCESSOR_EXECUTOR_POOL,
+ ThreadName.PIPE_PROCESSOR_EXECUTOR_POOL.getName(),
false);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
index f0de4d8d58e..e0a71454d23 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSubtaskExecutorManager.java
@@ -23,42 +23,44 @@ import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.db.pipe.consensus.PipeConsensusSubtaskExecutor;
import org.apache.iotdb.db.subscription.task.execution.SubscriptionSubtaskExecutor;
+import java.util.function.Supplier;
+
/**
* PipeTaskExecutor is responsible for executing the pipe tasks, and it is scheduled by the
* PipeTaskScheduler. It is a singleton class.
*/
public class PipeSubtaskExecutorManager {
private final PipeProcessorSubtaskExecutor processorExecutor;
- private final PipeConnectorSubtaskExecutor connectorExecutor;
+ private final Supplier<PipeConnectorSubtaskExecutor> connectorExecutorSupplier;
private final SubscriptionSubtaskExecutor subscriptionExecutor;
- private final PipeConsensusSubtaskExecutor consensusExecutor;
+ private final Supplier<PipeConsensusSubtaskExecutor> consensusExecutorSupplier;
public PipeProcessorSubtaskExecutor getProcessorExecutor() {
return processorExecutor;
}
- public PipeConnectorSubtaskExecutor getConnectorExecutor() {
- return connectorExecutor;
+ public Supplier<PipeConnectorSubtaskExecutor> getConnectorExecutorSupplier() {
+ return connectorExecutorSupplier;
}
public SubscriptionSubtaskExecutor getSubscriptionExecutor() {
return subscriptionExecutor;
}
- public PipeConsensusSubtaskExecutor getConsensusExecutor() {
- return consensusExecutor;
+ public Supplier<PipeConsensusSubtaskExecutor> getConsensusExecutorSupplier() {
+ return consensusExecutorSupplier;
}
///////////////////////// Singleton Instance Holder /////////////////////////
private PipeSubtaskExecutorManager() {
processorExecutor = new PipeProcessorSubtaskExecutor();
- connectorExecutor = new PipeConnectorSubtaskExecutor();
+ connectorExecutorSupplier = PipeConnectorSubtaskExecutor::new;
subscriptionExecutor =
SubscriptionConfig.getInstance().getSubscriptionEnabled()
? new SubscriptionSubtaskExecutor()
: null;
- consensusExecutor = new PipeConsensusSubtaskExecutor();
+ consensusExecutorSupplier = PipeConsensusSubtaskExecutor::new;
}
private static class PipeTaskExecutorHolder {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java
index 345e6ab040c..e5ae5dacd27 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskConnectorStage.java
@@ -28,13 +28,15 @@ import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.exception.PipeException;
+import java.util.function.Supplier;
+
public class PipeTaskConnectorStage extends PipeTaskStage {
protected final String pipeName;
protected final long creationTime;
protected final PipeParameters pipeConnectorParameters;
protected final int regionId;
- protected final PipeConnectorSubtaskExecutor executor;
+ protected final Supplier<? extends PipeConnectorSubtaskExecutor> executor;
protected String connectorSubtaskId;
@@ -43,7 +45,7 @@ public class PipeTaskConnectorStage extends PipeTaskStage {
long creationTime,
PipeParameters pipeConnectorParameters,
int regionId,
- PipeConnectorSubtaskExecutor executor) {
+ Supplier<? extends PipeConnectorSubtaskExecutor> executor) {
this.pipeName = pipeName;
this.creationTime = creationTime;
this.pipeConnectorParameters = pipeConnectorParameters;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
index ecbbc641e4b..6e4a858a370 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskLifeCycle.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.connector;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.pipe.api.event.Event;
import org.slf4j.Logger;
@@ -65,6 +66,8 @@ public class PipeConnectorSubtaskLifeCycle implements AutoCloseable {
if (registeredTaskCount == 0) {
executor.register(subtask);
runningTaskCount = 0;
+
+ PipeDataNodeResourceManager.compaction().registerPipeConnectorSubtaskLifeCycle(this);
}
registeredTaskCount++;
@@ -152,5 +155,7 @@ public class PipeConnectorSubtaskLifeCycle implements AutoCloseable {
@Override
public synchronized void close() {
executor.deregister(subtask.getTaskID());
+
+ PipeDataNodeResourceManager.compaction().deregisterPipeConnectorSubtaskLifeCycle(this);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java
index dd461e7f952..9e867d6083e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java
@@ -50,6 +50,7 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
public class PipeConnectorSubtaskManager {
@@ -62,7 +63,7 @@ public class PipeConnectorSubtaskManager {
attributeSortedString2SubtaskLifeCycleMap = new HashMap<>();
public synchronized String register(
- final PipeConnectorSubtaskExecutor executor,
+ final Supplier<? extends PipeConnectorSubtaskExecutor> executorSupplier,
final PipeParameters pipeConnectorParameters,
final PipeTaskConnectorRuntimeEnvironment environment) {
final String connectorKey =
@@ -112,6 +113,8 @@ public class PipeConnectorSubtaskManager {
environment.setAttributeSortedString(attributeSortedString);
if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
+ final PipeConnectorSubtaskExecutor executor = executorSupplier.get();
+
final List<PipeConnectorSubtaskLifeCycle> pipeConnectorSubtaskLifeCycleList =
new ArrayList<>(connectorNum);
@@ -172,6 +175,11 @@ public class PipeConnectorSubtaskManager {
pipeConnectorSubtaskLifeCycleList.add(pipeConnectorSubtaskLifeCycle);
}
+ LOGGER.info(
+ "Pipe connector subtasks with attributes {} is bounded with connectorExecutor {} and callbackExecutor {}.",
+ attributeSortedString,
+ executor.getWorkingThreadName(),
+ executor.getCallbackThreadName());
attributeSortedString2SubtaskLifeCycleMap.put(
attributeSortedString, pipeConnectorSubtaskLifeCycleList);
}
@@ -195,10 +203,19 @@ public class PipeConnectorSubtaskManager {
final List<PipeConnectorSubtaskLifeCycle> lifeCycles =
attributeSortedString2SubtaskLifeCycleMap.get(attributeSortedString);
+
+ // Shall not be empty
+ final PipeConnectorSubtaskExecutor executor = lifeCycles.get(0).executor;
+
lifeCycles.removeIf(o -> o.deregister(pipeName, regionId));
if (lifeCycles.isEmpty()) {
attributeSortedString2SubtaskLifeCycleMap.remove(attributeSortedString);
+ executor.shutdown();
+ LOGGER.info(
+ "The executor {} and {} has been successfully shutdown.",
+ executor.getWorkingThreadName(),
+ executor.getCallbackThreadName());
}
PipeEventCommitManager.getInstance().deregister(pipeName, creationTime, regionId);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
index a4f05447eae..409447b5b2f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
@@ -21,23 +21,40 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.connector;
import org.apache.iotdb.commons.pipe.agent.task.connection.BlockingPendingQueue;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
+import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeCompactedTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQueue<Event> {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(PipeRealtimePriorityBlockingQueue.class);
+
private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
private final BlockingDeque<TsFileInsertionEvent> tsfileInsertEventDeque =
@@ -184,6 +201,138 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
return tsfileInsertEventDeque.peek();
}
+ public synchronized void replace(
+ String dataRegionId, Set<TsFileResource> sourceFiles, List<TsFileResource> targetFiles) {
+
+ final int regionId = Integer.parseInt(dataRegionId);
+ final Map<CommitterKey, Set<PipeTsFileInsertionEvent>> eventsToBeRemovedGroupByCommitterKey =
+ tsfileInsertEventDeque.stream()
+ .filter(
+ event ->
+ event instanceof PipeTsFileInsertionEvent
+ && ((PipeTsFileInsertionEvent) event).getRegionId() == regionId)
+ .map(event -> (PipeTsFileInsertionEvent) event)
+ .collect(
+ Collectors.groupingBy(
+ PipeTsFileInsertionEvent::getCommitterKey, Collectors.toSet()))
+ .entrySet()
+ .stream()
+ // Replace if all source files are present in the queue
+ .filter(entry -> entry.getValue().size() == sourceFiles.size())
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ if (eventsToBeRemovedGroupByCommitterKey.isEmpty()) {
+ LOGGER.info(
+ "Region {}: No TsFileInsertionEvents to replace for source files {}",
+ regionId,
+ sourceFiles.stream()
+ .map(TsFileResource::getTsFilePath)
+ .collect(Collectors.joining(", ")));
+ return;
+ }
+
+ final Map<CommitterKey, Set<PipeTsFileInsertionEvent>> eventsToBeAddedGroupByCommitterKey =
+ new HashMap<>();
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
+ eventsToBeRemovedGroupByCommitterKey.entrySet()) {
+ final CommitterKey committerKey = entry.getKey();
+ final PipeTsFileInsertionEvent anyEvent = entry.getValue().stream().findFirst().orElse(null);
+ final Set<PipeTsFileInsertionEvent> newEvents = new HashSet<>();
+ for (int i = 0; i < targetFiles.size(); i++) {
+ newEvents.add(
+ new PipeCompactedTsFileInsertionEvent(
+ committerKey,
+ entry.getValue(),
+ anyEvent,
+ targetFiles.get(i),
+ i == targetFiles.size() - 1));
+ }
+ eventsToBeAddedGroupByCommitterKey.put(committerKey, newEvents);
+ }
+
+ // Handling new events
+ final Set<PipeTsFileInsertionEvent> successfullyReferenceIncreasedEvents = new HashSet<>();
+ final AtomicBoolean
+ allSuccess = // To track if all events successfully increased the reference count
+ new AtomicBoolean(true);
+ outerLoop:
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> committerKeySetEntry :
+ eventsToBeAddedGroupByCommitterKey.entrySet()) {
+ for (final PipeTsFileInsertionEvent event : committerKeySetEntry.getValue()) {
+ if (event != null) {
+ try {
+ if (!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName())) {
+ allSuccess.set(false);
+ break outerLoop;
+ } else {
+ successfullyReferenceIncreasedEvents.add(event);
+ }
+ } catch (final Exception e) {
+ allSuccess.set(false);
+ break outerLoop;
+ }
+ }
+ }
+ }
+ if (!allSuccess.get()) {
+ // If any event failed to increase the reference count,
+ // we need to decrease the reference count for all successfully increased events
+ for (final PipeTsFileInsertionEvent event : successfullyReferenceIncreasedEvents) {
+ try {
+ event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to decrease reference count for event {} in PipeRealtimePriorityBlockingQueue",
+ event,
+ e);
+ }
+ }
+ return; // Exit early if any event failed to increase the reference count
+ } else {
+ // If all events successfully increased reference count,
+ // we can proceed to add them to the deque
+ for (final PipeTsFileInsertionEvent event : successfullyReferenceIncreasedEvents) {
+ tsfileInsertEventDeque.add(event);
+ eventCounter.increaseEventCount(event);
+ }
+ }
+
+ // Handling old events
+ for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
+ eventsToBeRemovedGroupByCommitterKey.entrySet()) {
+ for (final PipeTsFileInsertionEvent event : entry.getValue()) {
+ if (event != null) {
+ try {
+ event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), false);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to decrease reference count for event {} in PipeRealtimePriorityBlockingQueue",
+ event,
+ e);
+ }
+ eventCounter.decreaseEventCount(event);
+ }
+ }
+ }
+ final Set<PipeTsFileInsertionEvent> eventsToRemove = new HashSet<>();
+ for (Set<PipeTsFileInsertionEvent> pipeTsFileInsertionEvents :
+ eventsToBeRemovedGroupByCommitterKey.values()) {
+ eventsToRemove.addAll(pipeTsFileInsertionEvents);
+ }
+ tsfileInsertEventDeque.removeIf(eventsToRemove::contains);
+
+ LOGGER.info(
+ "Region {}: Replaced TsFileInsertionEvents {} with {}",
+ regionId,
+ eventsToBeRemovedGroupByCommitterKey.values().stream()
+ .flatMap(Set::stream)
+ .map(PipeTsFileInsertionEvent::coreReportMessage)
+ .collect(Collectors.joining(", ")),
+ eventsToBeAddedGroupByCommitterKey.values().stream()
+ .flatMap(Set::stream)
+ .map(PipeTsFileInsertionEvent::coreReportMessage)
+ .collect(Collectors.joining(", ")));
+ }
+
@Override
public void clear() {
super.clear();
@@ -244,7 +393,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
return tsfileInsertEventDeque.size();
}
- public void setOfferTsFileCounter(AtomicInteger offerTsFileCounter) {
+ public synchronized void setOfferTsFileCounter(AtomicInteger offerTsFileCounter) {
this.offerTsFileCounter = offerTsFileCounter;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java
index a2f75bf110a..75d6a08a664 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeAsyncClientManager.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.commons.client.ClientPoolFactory;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient;
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.connector.client.IoTDBClientManager;
@@ -48,7 +50,9 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_LOAD_BALANCE_PRIORITY_STRATEGY;
@@ -70,7 +74,12 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
// receiverAttributes -> IClientManager<TEndPoint, AsyncPipeDataTransferServiceClient>
private static final Map<String, IClientManager<TEndPoint, AsyncPipeDataTransferServiceClient>>
ASYNC_PIPE_DATA_TRANSFER_CLIENT_MANAGER_HOLDER = new ConcurrentHashMap<>();
+ private static final Map<String, ExecutorService> TS_FILE_ASYNC_EXECUTOR_HOLDER =
+ new ConcurrentHashMap<>();
+ private static final AtomicInteger id = new AtomicInteger(0);
+
private final IClientManager<TEndPoint, AsyncPipeDataTransferServiceClient> endPoint2Client;
+ private ExecutorService executor;
private final LoadBalancer loadBalancer;
@@ -123,6 +132,17 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
}
endPoint2Client = ASYNC_PIPE_DATA_TRANSFER_CLIENT_MANAGER_HOLDER.get(receiverAttributes);
+ if (isTSFileUsed) {
+ if (!TS_FILE_ASYNC_EXECUTOR_HOLDER.containsKey(receiverAttributes)) {
+ TS_FILE_ASYNC_EXECUTOR_HOLDER.putIfAbsent(
+ receiverAttributes,
+ IoTDBThreadPoolFactory.newFixedThreadPool(
+ PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize(),
+ ThreadName.PIPE_TSFILE_ASYNC_SEND_POOL.getName() + "-" + id.getAndIncrement()));
+ }
+ executor = TS_FILE_ASYNC_EXECUTOR_HOLDER.get(receiverAttributes);
+ }
+
RECEIVER_ATTRIBUTES_REF_COUNT.compute(
receiverAttributes, (attributes, refCount) -> refCount == null ? 1 : refCount + 1);
}
@@ -348,6 +368,10 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
LEADER_CACHE_MANAGER.updateLeaderEndPoint(deviceId, endPoint);
}
+ public ExecutorService getExecutor() {
+ return executor;
+ }
+
public void close() {
isClosed = true;
synchronized (IoTDBDataNodeAsyncClientManager.class) {
@@ -370,6 +394,18 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
e);
}
}
+
+ final ExecutorService executor =
+ TS_FILE_ASYNC_EXECUTOR_HOLDER.remove(receiverAttributes);
+ if (executor != null) {
+ try {
+ executor.shutdown();
+ LOGGER.info("Successfully shutdown executor {}.", executor);
+ } catch (final Exception e) {
+ LOGGER.warn("Failed to shutdown executor {}.", executor);
+ }
+ }
+
return null;
}
return refCount - 1;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
index 1cb9f50d92d..8d52055fcc9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventBatch.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.pipe.connector.payload.evolvable.batch;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.memory.PipeDynamicMemoryBlock;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlockType;
import org.apache.iotdb.db.pipe.resource.memory.PipeModelFixedMemoryBlock;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
@@ -47,7 +48,7 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
private long firstEventProcessingTime = Long.MIN_VALUE;
protected long totalBufferSize = 0;
- private final PipeModelFixedMemoryBlock allocatedMemoryBlock;
+ private final PipeDynamicMemoryBlock allocatedMemoryBlock;
protected volatile boolean isClosed = false;
@@ -60,11 +61,9 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
// limit in buffer size
this.allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(
- requestMaxBatchSizeInBytes, PipeMemoryBlockType.BATCH);
+ pipeModelFixedMemoryBlock.registerPipeBatchMemoryBlock(requestMaxBatchSizeInBytes);
- if (getMaxBatchSizeInBytes() != requestMaxBatchSizeInBytes) {
+ if (getMaxBatchSizeInBytes() != allocatedMemoryBlock.getMemoryUsageInBytes()) {
LOGGER.info(
"PipeTabletEventBatch: the max batch size is adjusted from {} to {} due to the "
+ "memory restriction",
@@ -127,10 +126,17 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
throws WALPipeException, IOException;
public boolean shouldEmit() {
- return totalBufferSize >= getMaxBatchSizeInBytes()
- || System.currentTimeMillis() - firstEventProcessingTime >= maxDelayInMs;
+ final long diff = System.currentTimeMillis() - firstEventProcessingTime;
+ if (totalBufferSize >= getMaxBatchSizeInBytes() || diff >= maxDelayInMs) {
+ allocatedMemoryBlock.updateCurrentMemoryEfficiencyAdjustMem((double) diff / maxDelayInMs);
+ recordMetric(diff, totalBufferSize);
+ return true;
+ }
+ return false;
}
+ protected abstract void recordMetric(final long timeInterval, final long bufferSize);
+
private long getMaxBatchSizeInBytes() {
return allocatedMemoryBlock.getMemoryUsageInBytes();
}
@@ -194,9 +200,22 @@ public abstract class PipeTabletEventBatch implements AutoCloseable {
}
try {
- pipeModelFixedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(0L, PipeMemoryBlockType.BATCH);
+ long batchSize = PipeDataNodeResourceManager.memory().getAllocatedMemorySizeInBytesOfBatch();
+ for (long i = batchSize; i > 0; i = i / 2) {
+ try {
+ pipeModelFixedMemoryBlock =
+ PipeDataNodeResourceManager.memory()
+ .forceAllocateForModelFixedMemoryBlock(i, PipeMemoryBlockType.BATCH);
+
+ LOGGER.info("pipe model fixed memory block initialized with size: {} bytes", i);
+ return;
+ } catch (Exception ignore) {
+ // ignore the exception and try to allocate a smaller size
+ LOGGER.info(
+ "pipe model fixed memory block initialized with size: {} bytes failed, try smaller size",
+ i);
+ }
+ }
} catch (Exception e) {
LOGGER.error("init pipe model fixed memory block failed", e);
// If the allocation fails, we still need to create a default memory block to avoid NPE.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
index 572fe26827d..50480537cf9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletBatchReqV2;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
@@ -71,6 +72,12 @@ public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
return true;
}
+ @Override
+ protected void recordMetric(long timeInterval, long bufferSize) {
+ PipeDataRegionConnectorMetrics.tabletBatchTimeIntervalHistogram.update(timeInterval);
+ PipeDataRegionConnectorMetrics.tabletBatchSizeHistogram.update(bufferSize);
+ }
+
@Override
public synchronized void onSuccess() {
super.onSuccess();
@@ -113,8 +120,7 @@ public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
(PipeInsertNodeTabletInsertionEvent) event;
// Read the bytebuffer from the wal file and transfer it directly without serializing or
// deserializing if possible
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
if (Objects.isNull(insertNode)) {
buffer = pipeInsertNodeTabletInsertionEvent.getByteBuffer();
binaryBuffers.add(buffer);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
index 8eab2c6453c..d1b5a57259d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTableModelTabletEventS
import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTreeModelTabletEventSorter;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
@@ -125,6 +126,12 @@ public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch {
return true;
}
+ @Override
+ protected void recordMetric(long timeInterval, long bufferSize) {
+ PipeDataRegionConnectorMetrics.tsFileBatchTimeIntervalHistogram.update(timeInterval);
+ PipeDataRegionConnectorMetrics.tsFileBatchSizeHistogram.update(bufferSize);
+ }
+
private void bufferTreeModelTablet(
final String pipeName,
final long creationTime,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
index 9f27e4b4b48..e77d521b53f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferPlanNodeReq.java
@@ -66,7 +66,6 @@ public class PipeTransferPlanNodeReq extends TPipeTransferReq {
planNodeReq.version = transferReq.version;
planNodeReq.type = transferReq.type;
- planNodeReq.body = transferReq.body;
return planNodeReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
index 8090f650489..27664cdbac7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBatchReq.java
@@ -182,7 +182,6 @@ public class PipeTransferTabletBatchReq extends TPipeTransferReq {
batchReq.version = transferReq.version;
batchReq.type = transferReq.type;
- batchReq.body = transferReq.body;
return batchReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
index 41427238c7f..36390c649b4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletBinaryReq.java
@@ -93,7 +93,6 @@ public class PipeTransferTabletBinaryReq extends TPipeTransferReq {
binaryReq.version = transferReq.version;
binaryReq.type = transferReq.type;
- binaryReq.body = transferReq.body;
return binaryReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
index d5bd8e94f70..5944b3a368a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletInsertNodeReq.java
@@ -97,7 +97,6 @@ public class PipeTransferTabletInsertNodeReq extends TPipeTransferReq {
insertNodeReq.version = transferReq.version;
insertNodeReq.type = transferReq.type;
- insertNodeReq.body = transferReq.body;
return insertNodeReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
index bf27b697698..0109feb5ee0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java
@@ -132,7 +132,6 @@ public class PipeTransferTabletRawReq extends TPipeTransferReq {
tabletReq.version = transferReq.version;
tabletReq.type = transferReq.type;
- tabletReq.body = transferReq.body;
return tabletReq;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
index 4e81944a057..b2520e61313 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java
@@ -232,8 +232,7 @@ public class IoTDBDataRegionAirGapConnector extends IoTDBDataNodeAirGapConnector
final AirGapSocket socket,
final PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent)
throws PipeException, WALPipeException, IOException {
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final byte[] bytes =
Objects.isNull(insertNode)
? PipeTransferTabletBinaryReqV2.toTPipeTransferBytes(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
index bf1dab239f9..efa7c421c1c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusAsyncConnector.java
@@ -313,8 +313,7 @@ public class PipeConsensusAsyncConnector extends IoTDBConnector implements Conse
pipeInsertNodeTabletInsertionEvent.getCommitterKey().getRestartTimes(),
pipeInsertNodeTabletInsertionEvent.getRebootTimes());
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final ProgressIndex progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
final TPipeConsensusTransferReq pipeConsensusTransferReq =
Objects.isNull(insertNode)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
index 02a632ec84d..f501e8af7ba 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/PipeConsensusSyncConnector.java
@@ -325,7 +325,7 @@ public class PipeConsensusSyncConnector extends IoTDBConnector {
try (final SyncPipeConsensusServiceClient syncPipeConsensusServiceClient =
syncRetryClientManager.borrowClient(getFollowerUrl())) {
- insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
if (insertNode != null) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
index 978646fe8fb..b6f555b4ebc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/builder/PipeConsensusTransferBatchReqBuilder.java
@@ -198,8 +198,7 @@ public abstract class PipeConsensusTransferBatchReqBuilder implements AutoClosea
// Read the bytebuffer from the wal file and transfer it directly without serializing or
// deserializing if possible
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
// PipeConsensus will transfer binary data to TPipeConsensusTransferReq
final ProgressIndex progressIndex = pipeInsertNodeTabletInsertionEvent.getProgressIndex();
if (Objects.isNull(insertNode)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
index 176bcd94cfb..1b743530421 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
@@ -76,8 +76,6 @@ import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -104,12 +102,11 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
"Exception occurred while sending to receiver %s:%s.";
private static final boolean isSplitTSFileBatchModeEnabled = true;
- private static final ExecutorService executor =
- Executors.newFixedThreadPool(PipeConfig.getInstance().getPipeAsyncConnectorMaxClientNumber());
private final IoTDBDataRegionSyncConnector syncConnector = new IoTDBDataRegionSyncConnector();
private final BlockingQueue<Event> retryEventQueue = new LinkedBlockingQueue<>();
+ private final BlockingQueue<TsFileInsertionEvent> retryTsFileQueue = new LinkedBlockingQueue<>();
private final PipeDataRegionEventCounter retryEventQueueEventCounter =
new PipeDataRegionEventCounter();
@@ -284,8 +281,7 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
return false;
}
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final String databaseName =
pipeInsertNodeTabletInsertionEvent.isTableModelEvent()
? pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName()
@@ -447,9 +443,9 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
}
return null;
},
- executor);
+ transferTsFileClientManager.getExecutor());
- if (PipeConfig.getInstance().isTransferTsFileSync()) {
+ if (PipeConfig.getInstance().isTransferTsFileSync() || !isRealtimeFirst) {
try {
completableFuture.get();
} catch (InterruptedException e) {
@@ -527,7 +523,7 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
* @see PipeConnector#transfer(TsFileInsertionEvent) for more details.
*/
private void transferQueuedEventsIfNecessary(final boolean forced) {
- if (retryEventQueue.isEmpty()
+ if ((retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty())
|| (!forced
&& retryEventQueueEventCounter.getTabletInsertionEventCount()
< PipeConfig.getInstance()
@@ -535,38 +531,49 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
&& retryEventQueueEventCounter.getTsFileInsertionEventCount()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold()
- && retryEventQueue.size()
+ && retryEventQueue.size() + retryTsFileQueue.size()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold())) {
return;
}
final long retryStartTime = System.currentTimeMillis();
- final int remainingEvents = retryEventQueue.size();
- while (!retryEventQueue.isEmpty()) {
+ final int remainingEvents = retryEventQueue.size() + retryTsFileQueue.size();
+ while (!retryEventQueue.isEmpty() || !retryTsFileQueue.isEmpty()) {
synchronized (this) {
if (isClosed.get()) {
return;
}
- if (retryEventQueue.isEmpty()) {
+ if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
break;
}
- final Event peekedEvent = retryEventQueue.peek();
+ final Event peekedEvent;
+ final Event polledEvent;
+ if (!retryEventQueue.isEmpty()) {
+ peekedEvent = retryEventQueue.peek();
+
+ if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) {
+ retryTransfer((PipeInsertNodeTabletInsertionEvent) peekedEvent);
+ } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) {
+ retryTransfer((PipeRawTabletInsertionEvent) peekedEvent);
+ } else {
+ LOGGER.warn(
+ "IoTDBThriftAsyncConnector does not support transfer generic event: {}.",
+ peekedEvent);
+ }
- if (peekedEvent instanceof PipeInsertNodeTabletInsertionEvent) {
- retryTransfer((PipeInsertNodeTabletInsertionEvent) peekedEvent);
- } else if (peekedEvent instanceof PipeRawTabletInsertionEvent) {
- retryTransfer((PipeRawTabletInsertionEvent) peekedEvent);
- } else if (peekedEvent instanceof PipeTsFileInsertionEvent) {
- retryTransfer((PipeTsFileInsertionEvent) peekedEvent);
+ polledEvent = retryEventQueue.poll();
} else {
- LOGGER.warn(
- "IoTDBThriftAsyncConnector does not support transfer generic event: {}.",
- peekedEvent);
+ if (transferTsFileCounter.get()
+ >= PipeConfig.getInstance().getPipeRealTimeQueueMaxWaitingTsFileSize()) {
+ return;
+ }
+ peekedEvent = retryTsFileQueue.peek();
+ retryTransfer((PipeTsFileInsertionEvent) peekedEvent);
+ polledEvent = retryTsFileQueue.poll();
}
- final Event polledEvent = retryEventQueue.poll();
retryEventQueueEventCounter.decreaseEventCount(polledEvent);
if (polledEvent != peekedEvent) {
LOGGER.error(
@@ -589,16 +596,16 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
&& retryEventQueueEventCounter.getTsFileInsertionEventCount()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold()
- && retryEventQueue.size()
+ && retryEventQueue.size() + retryTsFileQueue.size()
< PipeConfig.getInstance()
.getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold()) {
return;
}
- if (remainingEvents <= retryEventQueue.size()) {
+ if (remainingEvents <= retryEventQueue.size() + retryTsFileQueue.size()) {
throw new PipeException(
"Failed to retry transferring events in the retry queue. Remaining events: "
- + retryEventQueue.size()
+ + (retryEventQueue.size() + retryTsFileQueue.size())
+ " (tablet events: "
+ retryEventQueueEventCounter.getTabletInsertionEventCount()
+ ", tsfile events: "
@@ -676,8 +683,14 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
return;
}
- retryEventQueue.offer(event);
- retryEventQueueEventCounter.increaseEventCount(event);
+ if (event instanceof PipeTsFileInsertionEvent) {
+ retryTsFileQueue.offer((PipeTsFileInsertionEvent) event);
+ retryEventQueueEventCounter.increaseEventCount(event);
+ } else {
+ retryEventQueue.offer(event);
+ retryEventQueueEventCounter.increaseEventCount(event);
+ }
+
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Added event {} to retry queue.", event);
}
@@ -721,6 +734,19 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
}
return false;
});
+
+ retryTsFileQueue.removeIf(
+ event -> {
+ if (event instanceof EnrichedEvent
+ && pipeNameToDrop.equals(((EnrichedEvent) event).getPipeName())
+ && regionId == ((EnrichedEvent) event).getRegionId()) {
+ ((EnrichedEvent) event)
+ .clearReferenceCount(IoTDBDataRegionAsyncConnector.class.getName());
+ retryEventQueueEventCounter.decreaseEventCount(event);
+ return true;
+ }
+ return false;
+ });
}
@Override
@@ -763,8 +789,9 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
}
public synchronized void clearRetryEventsReferenceCount() {
- while (!retryEventQueue.isEmpty()) {
- final Event event = retryEventQueue.poll();
+ while (!retryEventQueue.isEmpty() || !retryTsFileQueue.isEmpty()) {
+ final Event event =
+ retryTsFileQueue.isEmpty() ? retryEventQueue.poll() : retryTsFileQueue.poll();
retryEventQueueEventCounter.decreaseEventCount(event);
if (event instanceof EnrichedEvent) {
((EnrichedEvent) event).clearReferenceCount(IoTDBDataRegionAsyncConnector.class.getName());
@@ -775,7 +802,7 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
//////////////////////// APIs provided for metric framework ////////////////////////
public int getRetryEventQueueSize() {
- return retryEventQueue.size();
+ return retryEventQueue.size() + retryTsFileQueue.size();
}
public int getBatchSize() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
index 46ff84276b0..f3b48a6ff8a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java
@@ -48,7 +48,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Arrays;
@@ -81,8 +80,8 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
private final String dataBaseName;
private final int readFileBufferSize;
- private final PipeTsFileMemoryBlock memoryBlock;
- private final byte[] readBuffer;
+ private PipeTsFileMemoryBlock memoryBlock;
+ private byte[] readBuffer;
private long position;
private RandomAccessFile reader;
@@ -102,7 +101,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
final File modFile,
final boolean transferMod,
final String dataBaseName)
- throws FileNotFoundException, InterruptedException {
+ throws InterruptedException {
super(connector);
this.pipeName2WeightMap = pipeName2WeightMap;
@@ -129,20 +128,8 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
Math.min(
PipeConfig.getInstance().getPipeConnectorReadFileBufferSize(),
transferMod ? Math.max(tsFile.length(), modFile.length()) : tsFile.length());
- memoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForTsFileWithRetry(
- PipeConfig.getInstance().isPipeConnectorReadFileBufferMemoryControlEnabled()
- ? readFileBufferSize
- : 0);
- readBuffer = new byte[readFileBufferSize];
position = 0;
- reader =
- Objects.nonNull(modFile)
- ? new RandomAccessFile(modFile, "r")
- : new RandomAccessFile(tsFile, "r");
-
isSealSignalSent = new AtomicBoolean(false);
}
@@ -150,6 +137,24 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
final IoTDBDataNodeAsyncClientManager clientManager,
final AsyncPipeDataTransferServiceClient client)
throws TException, IOException {
+ // Delay creation of resources to avoid OOM or too many open files
+ if (readBuffer == null) {
+ memoryBlock =
+ PipeDataNodeResourceManager.memory()
+ .forceAllocateForTsFileWithRetry(
+ PipeConfig.getInstance().isPipeConnectorReadFileBufferMemoryControlEnabled()
+ ? readFileBufferSize
+ : 0);
+ readBuffer = new byte[readFileBufferSize];
+ }
+
+ if (reader == null) {
+ reader =
+ Objects.nonNull(modFile)
+ ? new RandomAccessFile(modFile, "r")
+ : new RandomAccessFile(tsFile, "r");
+ }
+
this.clientManager = clientManager;
this.client = client;
@@ -299,7 +304,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
"Successfully transferred file {} (committer key={}, commit id={}, reference count={}).",
tsFile,
events.stream().map(EnrichedEvent::getCommitterKey).collect(Collectors.toList()),
- events.stream().map(EnrichedEvent::getCommitId).collect(Collectors.toList()),
+ events.stream().map(EnrichedEvent::getCommitIds).collect(Collectors.toList()),
referenceCount);
} else {
LOGGER.info(
@@ -364,7 +369,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
"Failed to transfer TsFileInsertionEvent {} (committer key {}, commit id {}).",
tsFile,
events.stream().map(EnrichedEvent::getCommitterKey).collect(Collectors.toList()),
- events.stream().map(EnrichedEvent::getCommitId).collect(Collectors.toList()),
+ events.stream().map(EnrichedEvent::getCommitIds).collect(Collectors.toList()),
exception);
} else {
LOGGER.warn(
@@ -455,7 +460,9 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler {
@Override
public void close() {
super.close();
- memoryBlock.close();
+ if (memoryBlock != null) {
+ memoryBlock.close();
+ }
}
/**
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
index 5d222fd79ee..391b1769c2d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java
@@ -376,8 +376,7 @@ public class IoTDBDataRegionSyncConnector extends IoTDBDataNodeSyncConnector {
// getDeviceId() may return null for InsertRowsNode, will be equal to getClient(null)
clientAndStatus = clientManager.getClient(pipeInsertNodeTabletInsertionEvent.getDeviceId());
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final TPipeTransferReq req =
compressIfNeeded(
insertNode != null
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
index 8453798bda6..18a4feb3a3f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
@@ -230,8 +230,7 @@ public class WriteBackConnector implements PipeConnector {
private void doTransfer(
final PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent)
throws PipeException, WALPipeException, IOException {
- final InsertNode insertNode =
- pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = pipeInsertNodeTabletInsertionEvent.getInsertNode();
final String dataBaseName =
pipeInsertNodeTabletInsertionEvent.isTableModelEvent()
? pipeInsertNodeTabletInsertionEvent.getTableModelDatabaseName()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java
index 72782ad228e..d50e08463e2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/PipeConsensusSubtaskExecutor.java
@@ -22,11 +22,15 @@ import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
+import java.util.concurrent.atomic.AtomicInteger;
+
public class PipeConsensusSubtaskExecutor extends PipeConnectorSubtaskExecutor {
+ private static final AtomicInteger id = new AtomicInteger(0);
+
public PipeConsensusSubtaskExecutor() {
super(
PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(),
- ThreadName.PIPE_CONSENSUS_EXECUTOR_POOL);
+ ThreadName.PIPE_CONSENSUS_EXECUTOR_POOL + "-" + id.getAndIncrement());
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
index c28dba1aec6..9b091a88f51 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventT
import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreePatternParser;
import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
import org.apache.iotdb.db.pipe.resource.memory.PipeTabletMemoryBlock;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
@@ -48,8 +49,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalIn
import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName;
import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryPosition;
import org.apache.iotdb.pipe.api.access.Row;
import org.apache.iotdb.pipe.api.collector.RowCollector;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
@@ -82,23 +81,19 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
LoggerFactory.getLogger(PipeInsertNodeTabletInsertionEvent.class);
private static final long INSTANCE_SIZE =
RamUsageEstimator.shallowSizeOfInstance(PipeInsertNodeTabletInsertionEvent.class)
- + RamUsageEstimator.shallowSizeOfInstance(WALEntryHandler.class)
- + RamUsageEstimator.shallowSizeOfInstance(WALEntryPosition.class)
+ RamUsageEstimator.shallowSizeOfInstance(AtomicInteger.class)
+ RamUsageEstimator.shallowSizeOfInstance(AtomicBoolean.class)
+ RamUsageEstimator.shallowSizeOf(Boolean.class);
private static final long SET_SIZE = RamUsageEstimator.shallowSizeOfInstance(HashSet.class);
- private final WALEntryHandler walEntryHandler;
- private final boolean isAligned;
- private final boolean isGeneratedByPipe;
-
private final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock;
private volatile List<Tablet> tablets;
private List<TabletInsertionEventParser> eventParsers;
private final PartialPath devicePath;
+ private InsertNode insertNode;
+
private ProgressIndex progressIndex;
// Only useful for insertRows
@@ -109,21 +104,15 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
public PipeInsertNodeTabletInsertionEvent(
final Boolean isTableModel,
final String databaseNameFromDataRegion,
- final WALEntryHandler walEntryHandler,
+ final InsertNode insertNode,
final PartialPath devicePath,
- final Set<String> tableNames,
- final ProgressIndex progressIndex,
- final boolean isAligned,
- final boolean isGeneratedByPipe) {
+ final Set<String> tableNames) {
this(
isTableModel,
databaseNameFromDataRegion,
- walEntryHandler,
+ insertNode,
devicePath,
tableNames,
- progressIndex,
- isAligned,
- isGeneratedByPipe,
null,
0,
null,
@@ -138,12 +127,9 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
private PipeInsertNodeTabletInsertionEvent(
final Boolean isTableModelEvent,
final String databaseNameFromDataRegion,
- final WALEntryHandler walEntryHandler,
+ final InsertNode insertNode,
final PartialPath devicePath,
final Set<String> tableNames,
- final ProgressIndex progressIndex,
- final boolean isAligned,
- final boolean isGeneratedByPipe,
final String pipeName,
final long creationTime,
final PipeTaskMeta pipeTaskMeta,
@@ -165,31 +151,21 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
endTime,
isTableModelEvent,
databaseNameFromDataRegion);
- this.walEntryHandler = walEntryHandler;
+ this.insertNode = insertNode;
// Record device path here so there's no need to get it from InsertNode cache later.
this.devicePath = devicePath;
this.tableNames = tableNames;
- this.progressIndex = progressIndex;
- this.isAligned = isAligned;
- this.isGeneratedByPipe = isGeneratedByPipe;
+ this.progressIndex = insertNode.getProgressIndex();
this.allocatedMemoryBlock = new AtomicReference<>();
}
- public InsertNode getInsertNode() throws WALPipeException {
- return walEntryHandler.getInsertNode();
+ public InsertNode getInsertNode() {
+ return insertNode;
}
public ByteBuffer getByteBuffer() throws WALPipeException {
- return walEntryHandler.getByteBuffer();
- }
-
- // This method is a pre-determination of whether to use binary transfers.
- // If the insert node is null in cache, it means that we need to read the bytebuffer from the wal,
- // and when the pattern is default, we can transfer the bytebuffer directly without serializing or
- // deserializing
- public InsertNode getInsertNodeViaCacheIfPossible() {
- return walEntryHandler.getInsertNodeViaCacheIfPossible();
+ return insertNode.serializeToByteBuffer();
}
public String getDeviceId() {
@@ -202,19 +178,16 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
extractTime = System.nanoTime();
try {
- PipeDataNodeResourceManager.wal().pin(walEntryHandler);
if (Objects.nonNull(pipeName)) {
PipeDataNodeSinglePipeMetrics.getInstance()
.increaseInsertNodeEventCount(pipeName, creationTime);
- PipeDataNodeAgent.task().addFloatingMemoryUsageInByte(pipeName, ramBytesUsed());
+ PipeDataNodeAgent.task()
+ .addFloatingMemoryUsageInByte(pipeName, creationTime, ramBytesUsed());
}
return true;
} catch (final Exception e) {
LOGGER.warn(
- String.format(
- "Increase reference count for memTable %d error. Holder Message: %s",
- walEntryHandler.getMemTableId(), holderMessage),
- e);
+ String.format("Increase reference count error. Holder Message: %s", holderMessage), e);
return false;
}
}
@@ -222,7 +195,6 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
@Override
public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
try {
- PipeDataNodeResourceManager.wal().unpin(walEntryHandler);
// release the parsers' memory and close memory block
if (eventParsers != null) {
eventParsers.clear();
@@ -232,17 +204,16 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
return true;
} catch (final Exception e) {
LOGGER.warn(
- String.format(
- "Decrease reference count for memtable %d error. Holder Message: %s",
- walEntryHandler.getMemTableId(), holderMessage),
- e);
+ String.format("Decrease reference count error. Holder Message: %s", holderMessage), e);
return false;
} finally {
if (Objects.nonNull(pipeName)) {
- PipeDataNodeAgent.task().decreaseFloatingMemoryUsageInByte(pipeName, ramBytesUsed());
+ PipeDataNodeAgent.task()
+ .decreaseFloatingMemoryUsageInByte(pipeName, creationTime, ramBytesUsed());
PipeDataNodeSinglePipeMetrics.getInstance()
.decreaseInsertNodeEventCount(pipeName, creationTime, System.nanoTime() - extractTime);
}
+ insertNode = null;
}
}
@@ -270,12 +241,9 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
return new PipeInsertNodeTabletInsertionEvent(
getRawIsTableModelEvent(),
getSourceDatabaseNameFromDataRegion(),
- walEntryHandler,
+ insertNode,
devicePath,
tableNames,
- progressIndex,
- isAligned,
- isGeneratedByPipe,
pipeName,
creationTime,
pipeTaskMeta,
@@ -289,7 +257,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
@Override
public boolean isGeneratedByPipe() {
- return isGeneratedByPipe;
+ return insertNode.isGeneratedByPipe();
}
@Override
@@ -323,7 +291,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
@Override
public boolean mayEventTimeOverlappedWithTimeRange() {
try {
- final InsertNode insertNode = getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = getInsertNode();
if (Objects.isNull(insertNode)) {
return true;
}
@@ -367,7 +335,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
@Override
public boolean mayEventPathsOverlappedWithPattern() {
try {
- final InsertNode insertNode = getInsertNodeViaCacheIfPossible();
+ final InsertNode insertNode = getInsertNode();
if (Objects.isNull(insertNode)) {
return true;
}
@@ -545,8 +513,11 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
@Override
public String toString() {
return String.format(
- "PipeInsertNodeTabletInsertionEvent{walEntryHandler=%s, progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s, eventParsers=%s}",
- walEntryHandler, progressIndex, isAligned, isGeneratedByPipe, eventParsers)
+ "PipeInsertNodeTabletInsertionEvent{progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s, eventParsers=%s}",
+ progressIndex,
+ Objects.nonNull(insertNode) ? insertNode.isAligned() : null,
+ Objects.nonNull(insertNode) ? insertNode.isGeneratedByPipe() : null,
+ eventParsers)
+ " - "
+ super.toString();
}
@@ -554,8 +525,10 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
@Override
public String coreReportMessage() {
return String.format(
- "PipeInsertNodeTabletInsertionEvent{walEntryHandler=%s, progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s}",
- walEntryHandler, progressIndex, isAligned, isGeneratedByPipe)
+ "PipeInsertNodeTabletInsertionEvent{progressIndex=%s, isAligned=%s, isGeneratedByPipe=%s}",
+ progressIndex,
+ Objects.nonNull(insertNode) ? insertNode.isAligned() : null,
+ Objects.nonNull(insertNode) ? insertNode.isGeneratedByPipe() : null)
+ " - "
+ super.coreReportMessage();
}
@@ -570,7 +543,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
@Override
public PipeEventResource eventResourceBuilder() {
return new PipeInsertNodeTabletInsertionEventResource(
- this.isReleased, this.referenceCount, this.walEntryHandler, this.allocatedMemoryBlock);
+ this.isReleased, this.referenceCount, this.allocatedMemoryBlock);
}
// Notes:
@@ -582,6 +555,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
public long ramBytesUsed() {
return INSTANCE_SIZE
+ (Objects.nonNull(devicePath) ? PartialPath.estimateSize(devicePath) : 0)
+ + (Objects.nonNull(insertNode) ? InsertNodeMemoryEstimator.sizeOf(insertNode) : 0)
+ (Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0)
+ (Objects.nonNull(treeModelDatabaseName)
? RamUsageEstimator.sizeOf(treeModelDatabaseName)
@@ -597,23 +571,19 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
private static class PipeInsertNodeTabletInsertionEventResource extends PipeEventResource {
- private final WALEntryHandler walEntryHandler;
private final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock;
private PipeInsertNodeTabletInsertionEventResource(
final AtomicBoolean isReleased,
final AtomicInteger referenceCount,
- final WALEntryHandler walEntryHandler,
final AtomicReference<PipeTabletMemoryBlock> allocatedMemoryBlock) {
super(isReleased, referenceCount);
- this.walEntryHandler = walEntryHandler;
this.allocatedMemoryBlock = allocatedMemoryBlock;
}
@Override
protected void finalizeResource() {
try {
- PipeDataNodeResourceManager.wal().unpin(walEntryHandler);
allocatedMemoryBlock.getAndUpdate(
memoryBlock -> {
if (Objects.nonNull(memoryBlock)) {
@@ -622,8 +592,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
return null;
});
} catch (final Exception e) {
- LOGGER.warn(
- "Decrease reference count for memTable {} error.", walEntryHandler.getMemTableId(), e);
+ LOGGER.warn("Decrease reference count error.", e);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
index d431b12a79c..f85b26b6228 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java
@@ -19,24 +19,17 @@
package org.apache.iotdb.db.pipe.event.common.terminate;
-import org.apache.iotdb.common.rpc.thrift.TFlushReq;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTask;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
-import org.apache.iotdb.db.storageengine.StorageEngine;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicLong;
/**
* The {@link PipeTerminateEvent} is an {@link EnrichedEvent} that controls the termination of pipe,
@@ -46,40 +39,6 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class PipeTerminateEvent extends EnrichedEvent {
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeTerminateEvent.class);
-
- private static final AtomicLong PROGRESS_REPORT_COUNT = new AtomicLong(0);
- private static final AtomicLong LAST_PROGRESS_REPORT_TIME = new AtomicLong(0);
-
- public static void flushDataRegionIfNeeded() {
- if (PROGRESS_REPORT_COUNT.get() > 0
- && PROGRESS_REPORT_COUNT.get()
- > PipeConfig.getInstance().getPipeFlushAfterTerminateCount()) {
- flushDataRegion();
- return;
- }
-
- if (LAST_PROGRESS_REPORT_TIME.get() > 0
- && System.currentTimeMillis() - LAST_PROGRESS_REPORT_TIME.get()
- > PipeConfig.getInstance().getPipeFlushAfterLastTerminateSeconds() * 1000L) {
- flushDataRegion();
- }
- }
-
- private static void flushDataRegion() {
- try {
- StorageEngine.getInstance().operateFlush(new TFlushReq());
- PROGRESS_REPORT_COUNT.set(0);
- LAST_PROGRESS_REPORT_TIME.set(0);
- LOGGER.info("Force flush all data regions because of last progress report time.");
- } catch (final Exception e) {
- LOGGER.warn(
- "Failed to flush all data regions, please check the error message: {}",
- e.getMessage(),
- e);
- }
- }
-
private final int dataRegionId;
public PipeTerminateEvent(
@@ -148,9 +107,6 @@ public class PipeTerminateEvent extends EnrichedEvent {
@Override
public void reportProgress() {
- PROGRESS_REPORT_COUNT.incrementAndGet();
- LAST_PROGRESS_REPORT_TIME.set(System.currentTimeMillis());
-
// To avoid deadlock
CompletableFuture.runAsync(
() -> PipeDataNodeAgent.task().markCompleted(pipeName, dataRegionId));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
new file mode 100644
index 00000000000..bcc6c5facdd
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeCompactedTsFileInsertionEvent.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.event.common.tsfile;
+
+import org.apache.iotdb.commons.consensus.index.ProgressIndex;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
+import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
+import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import java.io.File;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class PipeCompactedTsFileInsertionEvent extends PipeTsFileInsertionEvent {
+
+ private final String dataRegionId;
+ private final Set<String> originFilePaths;
+ private final List<Long> commitIds;
+
+ public PipeCompactedTsFileInsertionEvent(
+ final CommitterKey committerKey,
+ final Set<PipeTsFileInsertionEvent> originalEvents,
+ final PipeTsFileInsertionEvent anyOfOriginalEvents,
+ final TsFileResource tsFileResource,
+ final boolean shouldReportProgress) {
+ super(
+ anyOfOriginalEvents.isTableModelEvent(),
+ anyOfOriginalEvents.getTreeModelDatabaseName(),
+ tsFileResource,
+ null,
+ bindIsWithMod(originalEvents),
+ bindIsLoaded(originalEvents),
+ bindIsGeneratedByHistoricalExtractor(originalEvents),
+ committerKey.getPipeName(),
+ committerKey.getCreationTime(),
+ anyOfOriginalEvents.getPipeTaskMeta(),
+ anyOfOriginalEvents.getTreePattern(),
+ anyOfOriginalEvents.getTablePattern(),
+ anyOfOriginalEvents.getUserName(),
+ anyOfOriginalEvents.isSkipIfNoPrivileges(),
+ anyOfOriginalEvents.getStartTime(),
+ anyOfOriginalEvents.getEndTime());
+
+ this.dataRegionId = String.valueOf(committerKey.getRegionId());
+ this.originFilePaths =
+ originalEvents.stream()
+ .map(PipeTsFileInsertionEvent::getTsFile)
+ .map(File::getPath)
+ .collect(Collectors.toSet());
+ this.commitIds =
+ originalEvents.stream()
+ .map(PipeTsFileInsertionEvent::getCommitId)
+ .distinct()
+ .collect(Collectors.toList());
+
+ // init fields of EnrichedEvent
+ this.committerKey = committerKey;
+ isPatternParsed = bindIsPatternParsed(originalEvents);
+ isTimeParsed = bindIsTimeParsed(originalEvents);
+ this.shouldReportOnCommit = shouldReportProgress;
+
+ // init fields of PipeTsFileInsertionEvent
+ flushPointCount = bindFlushPointCount(originalEvents);
+ overridingProgressIndex = bindOverridingProgressIndex(originalEvents);
+ }
+
+ private static boolean bindIsWithMod(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().anyMatch(PipeTsFileInsertionEvent::isWithMod);
+ }
+
+ private static boolean bindIsLoaded(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().anyMatch(PipeTsFileInsertionEvent::isLoaded);
+ }
+
+ private static boolean bindIsGeneratedByHistoricalExtractor(
+ Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream()
+ .anyMatch(PipeTsFileInsertionEvent::isGeneratedByHistoricalExtractor);
+ }
+
+ private static boolean bindIsTimeParsed(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().noneMatch(EnrichedEvent::shouldParseTime);
+ }
+
+ private static boolean bindIsPatternParsed(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream().noneMatch(EnrichedEvent::shouldParsePattern);
+ }
+
+ private static long bindFlushPointCount(Set<PipeTsFileInsertionEvent> originalEvents) {
+ return originalEvents.stream()
+ .mapToLong(
+ e ->
+ e.getFlushPointCount() == TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET
+ ? 0
+ : e.getFlushPointCount())
+ .sum();
+ }
+
+ private ProgressIndex bindOverridingProgressIndex(Set<PipeTsFileInsertionEvent> originalEvents) {
+ ProgressIndex overridingProgressIndex = MinimumProgressIndex.INSTANCE;
+ for (PipeTsFileInsertionEvent originalEvent : originalEvents) {
+ if (originalEvent.overridingProgressIndex != null) {
+ overridingProgressIndex =
+ overridingProgressIndex.updateToMinimumEqualOrIsAfterProgressIndex(
+ originalEvent.overridingProgressIndex);
+ }
+ }
+ return overridingProgressIndex != null
+ && !overridingProgressIndex.equals(MinimumProgressIndex.INSTANCE)
+ ? overridingProgressIndex
+ : null;
+ }
+
+ @Override
+ public int getRebootTimes() {
+ throw new UnsupportedOperationException(
+ "PipeCompactedTsFileInsertionEvent does not support getRebootTimes.");
+ }
+
+ @Override
+ public boolean hasMultipleCommitIds() {
+ return true;
+ }
+
+ @Override
+ public long getCommitId() {
+ // max of commitIds is used as the commit id for this event
+ return commitIds.stream()
+ .max(Long::compareTo)
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "No commit IDs found in PipeCompactedTsFileInsertionEvent."));
+ }
+
+ // return dummy events for each commit ID (except the max one)
+ @Override
+ public List<EnrichedEvent> getDummyEventsForCommitIds() {
+ return commitIds.stream()
+ .filter(commitId -> commitId != getCommitId())
+ .map(PipeCompactedTsFileInsertionDummyEvent::new)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List<Long> getCommitIds() {
+ return commitIds;
+ }
+
+ @Override
+ public boolean equalsInPipeConsensus(final Object o) {
+ throw new UnsupportedOperationException(
+ "PipeCompactedTsFileInsertionEvent does not support equalsInPipeConsensus.");
+ }
+
+ @Override
+ public void eliminateProgressIndex() {
+ if (Objects.isNull(overridingProgressIndex)) {
+ for (final String originFilePath : originFilePaths) {
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(dataRegionId, pipeName, originFilePath);
+ }
+ }
+ }
+
+ public class PipeCompactedTsFileInsertionDummyEvent extends EnrichedEvent {
+
+ private final long commitId;
+
+ public PipeCompactedTsFileInsertionDummyEvent(final long commitId) {
+ super(
+ PipeCompactedTsFileInsertionEvent.this.pipeName,
+ PipeCompactedTsFileInsertionEvent.this.creationTime,
+ PipeCompactedTsFileInsertionEvent.this.pipeTaskMeta,
+ null, // PipePattern is not needed for dummy event
+ null,
+ null,
+ true,
+ Long.MIN_VALUE,
+ Long.MAX_VALUE);
+ this.commitId = commitId; // Use the commitId passed in
+ this.shouldReportOnCommit = false; // Dummy events do not report progress
+ }
+
+ @Override
+ public long getCommitId() {
+ return commitId;
+ }
+
+ @Override
+ public boolean internallyIncreaseResourceReferenceCount(String holderMessage) {
+ return true;
+ }
+
+ @Override
+ public boolean internallyDecreaseResourceReferenceCount(String holderMessage) {
+ return true;
+ }
+
+ @Override
+ public ProgressIndex getProgressIndex() {
+ return MinimumProgressIndex.INSTANCE;
+ }
+
+ @Override
+ public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+ final String pipeName,
+ final long creationTime,
+ final PipeTaskMeta pipeTaskMeta,
+ final TreePattern treePattern,
+ final TablePattern tablePattern,
+ final String userName,
+ final boolean skipIfNoPrivileges,
+ final long startTime,
+ final long endTime) {
+ return null;
+ }
+
+ @Override
+ public boolean isGeneratedByPipe() {
+ return false;
+ }
+
+ @Override
+ public boolean mayEventTimeOverlappedWithTimeRange() {
+ return false;
+ }
+
+ @Override
+ public boolean mayEventPathsOverlappedWithPattern() {
+ return false;
+ }
+
+ @Override
+ public String coreReportMessage() {
+ return "PipeCompactedTsFileInsertionDummyEvent";
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index f216ff51077..d791dddc713 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -72,7 +72,6 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
implements TsFileInsertionEvent, ReferenceTrackableEvent {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileInsertionEvent.class);
-
private static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR;
private final TsFileResource resource;
@@ -85,34 +84,33 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
private final File sharedModFile;
private boolean shouldParse4Privilege = false;
- private final boolean isLoaded;
- private final boolean isGeneratedByPipe;
- private final boolean isGeneratedByPipeConsensus;
- private final boolean isGeneratedByHistoricalExtractor;
-
+ protected final boolean isLoaded;
+ protected final boolean isGeneratedByPipe;
+ protected final boolean isGeneratedByPipeConsensus;
+ protected final boolean isGeneratedByHistoricalExtractor;
private final AtomicBoolean isClosed;
private final AtomicReference<TsFileInsertionEventParser> eventParser;
// The point count of the TsFile. Used for metrics on PipeConsensus' receiver side.
// May be updated after it is flushed. Should be negative if not set.
- private long flushPointCount = TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET;
+ protected long flushPointCount = TsFileProcessor.FLUSH_POINT_COUNT_NOT_SET;
- private volatile ProgressIndex overridingProgressIndex;
+ protected volatile ProgressIndex overridingProgressIndex;
public PipeTsFileInsertionEvent(
final Boolean isTableModelEvent,
final String databaseNameFromDataRegion,
final TsFileResource resource,
- final boolean isLoaded,
- final boolean isGeneratedByHistoricalExtractor) {
+ final boolean isLoaded) {
// The modFile must be copied before the event is assigned to the listening pipes
this(
isTableModelEvent,
databaseNameFromDataRegion,
resource,
+ null,
true,
isLoaded,
- isGeneratedByHistoricalExtractor,
+ false,
null,
0,
null,
@@ -128,6 +126,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
final Boolean isTableModelEvent,
final String databaseNameFromDataRegion,
final TsFileResource resource,
+ final File tsFile,
final boolean isWithMod,
final boolean isLoaded,
final boolean isGeneratedByHistoricalExtractor,
@@ -154,7 +153,12 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
databaseNameFromDataRegion);
this.resource = resource;
- tsFile = resource.getTsFile();
+
+ // For events created at assigner or historical extractor, the tsFile is get from the resource
+ // For events created for source, the tsFile is inherited from the assigner, because the
+ // original tsFile may be gone, and we need to get the assigner's hard-linked tsFile to
+ // hard-link it to each pipe dir
+ this.tsFile = Objects.isNull(tsFile) ? resource.getTsFile() : tsFile;
this.isWithMod = isWithMod && resource.anyModFileExists();
this.modFile = this.isWithMod ? resource.getExclusiveModFile().getFile() : null;
@@ -210,6 +214,10 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
* otherwise.
*/
public boolean waitForTsFileClose() throws InterruptedException {
+ if (Objects.isNull(resource)) {
+ return true;
+ }
+
if (!isClosed.get()) {
isClosed.set(resource.isClosed());
@@ -240,6 +248,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
return !resource.isEmpty();
}
+ @Override
public File getTsFile() {
return tsFile;
}
@@ -266,10 +275,6 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
return isLoaded;
}
- public long getFileStartTime() {
- return resource.getFileStartTime();
- }
-
/**
* Only used for metrics on PipeConsensus' receiver side. If the event is recovered after data
* node's restart, the flushPointCount can be not set. It's totally fine for the PipeConsensus'
@@ -292,9 +297,10 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) {
extractTime = System.nanoTime();
try {
- tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, resource);
+ tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, pipeName);
if (isWithMod) {
- modFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, null);
+ modFile =
+ PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, pipeName);
}
return true;
} catch (final Exception e) {
@@ -315,9 +321,9 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
@Override
public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) {
try {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName);
if (isWithMod) {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName);
}
close();
return true;
@@ -343,24 +349,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
@Override
public ProgressIndex getProgressIndex() {
- try {
- if (!waitForTsFileClose()) {
- LOGGER.warn(
- "Skipping temporary TsFile {}'s progressIndex, will report MinimumProgressIndex",
- tsFile);
- return MinimumProgressIndex.INSTANCE;
- }
- if (Objects.nonNull(overridingProgressIndex)) {
- return overridingProgressIndex;
- }
- return resource.getMaxProgressIndexAfterClose();
- } catch (final InterruptedException e) {
- LOGGER.warn(
- String.format(
- "Interrupted when waiting for closing TsFile %s.", resource.getTsFilePath()));
- Thread.currentThread().interrupt();
- return MinimumProgressIndex.INSTANCE;
- }
+ return resource.getMaxProgressIndex();
}
/**
@@ -386,9 +375,9 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
}
public void eliminateProgressIndex() {
- if (Objects.isNull(overridingProgressIndex)) {
+ if (Objects.isNull(overridingProgressIndex) && Objects.nonNull(resource)) {
PipeTsFileEpochProgressIndexKeeper.getInstance()
- .eliminateProgressIndex(resource.getDataRegionId(), resource.getTsFilePath());
+ .eliminateProgressIndex(resource.getDataRegionId(), pipeName, resource.getTsFilePath());
}
}
@@ -411,6 +400,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
getRawIsTableModelEvent(),
getSourceDatabaseNameFromDataRegion(),
resource,
+ tsFile,
isWithMod,
isLoaded,
isGeneratedByHistoricalExtractor,
@@ -482,16 +472,15 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
@Override
public boolean mayEventTimeOverlappedWithTimeRange() {
- // If the tsFile is not closed the resource.getFileEndTime() will be Long.MIN_VALUE
- // In that case we only judge the resource.getFileStartTime() to avoid losing data
- return isClosed.get()
- ? startTime <= resource.getFileEndTime() && resource.getFileStartTime() <= endTime
- : resource.getFileStartTime() <= endTime;
+ // Notice that this is only called at realtime extraction, and the tsFile is always closed
+ // Thus we can use the end time to judge the overlap
+ return Objects.isNull(resource)
+ || startTime <= resource.getFileEndTime() && resource.getFileStartTime() <= endTime;
}
@Override
public boolean mayEventPathsOverlappedWithPattern() {
- if (!resource.isClosed()) {
+ if (Objects.isNull(resource) || !resource.isClosed()) {
return true;
}
@@ -526,7 +515,8 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
final Map<IDeviceID, Boolean> deviceIsAlignedMap =
PipeDataNodeResourceManager.tsfile()
.getDeviceIsAlignedMapFromCache(
- PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()),
+ PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ resource.getTsFile(), pipeName),
false);
return Objects.nonNull(deviceIsAlignedMap)
? deviceIsAlignedMap.keySet()
@@ -726,7 +716,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
} catch (final IOException e) {
close();
- final String errorMsg = String.format("Read TsFile %s error.", resource.getTsFilePath());
+ final String errorMsg = String.format("Read TsFile %s error.", tsFile.getPath());
LOGGER.warn(errorMsg, e);
throw new PipeException(errorMsg, e);
}
@@ -783,8 +773,8 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
@Override
public String coreReportMessage() {
return String.format(
- "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s, isClosed=%s}",
- resource, tsFile, isLoaded, isGeneratedByPipe, isClosed.get())
+ "PipeTsFileInsertionEvent{resource=%s, tsFile=%s, isLoaded=%s, isGeneratedByPipe=%s}",
+ resource, tsFile, isLoaded, isGeneratedByPipe)
+ " - "
+ super.coreReportMessage();
}
@@ -801,6 +791,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
return new PipeTsFileInsertionEventResource(
this.isReleased,
this.referenceCount,
+ this.pipeName,
this.tsFile,
this.isWithMod,
this.modFile,
@@ -815,16 +806,19 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
private final File modFile;
private final File sharedModFile; // unused now
private final AtomicReference<TsFileInsertionEventParser> eventParser;
+ private final String pipeName;
private PipeTsFileInsertionEventResource(
final AtomicBoolean isReleased,
final AtomicInteger referenceCount,
+ final String pipeName,
final File tsFile,
final boolean isWithMod,
final File modFile,
final File sharedModFile,
final AtomicReference<TsFileInsertionEventParser> eventParser) {
super(isReleased, referenceCount);
+ this.pipeName = pipeName;
this.tsFile = tsFile;
this.isWithMod = isWithMod;
this.modFile = modFile;
@@ -836,9 +830,9 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent
protected void finalizeResource() {
try {
// decrease reference count
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName);
if (isWithMod) {
- PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile);
+ PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName);
}
// close event parser
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java
index 72c857c5c51..a965c817b99 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java
@@ -30,7 +30,7 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan.TsFileInsertionE
import org.apache.iotdb.db.pipe.event.common.tsfile.parser.table.TsFileInsertionEventTableParser;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResource;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFilePublicResource;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -100,7 +100,7 @@ public class TsFileInsertionEventParserProvider {
// Use scan container to save memory
if ((double) PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes()
/ PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- > PipeTsFileResource.MEMORY_SUFFICIENT_THRESHOLD) {
+ > PipeTsFilePublicResource.MEMORY_SUFFICIENT_THRESHOLD) {
return new TsFileInsertionEventScanParser(
pipeName,
creationTime,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
index cde00ee38d0..479d0fdf1f3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEventFactory.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
import java.util.stream.Collectors;
@@ -43,8 +42,7 @@ public class PipeRealtimeEventFactory {
final TsFileResource resource,
final boolean isLoaded) {
PipeTsFileInsertionEvent tsFileInsertionEvent =
- new PipeTsFileInsertionEvent(
- isTableModel, databaseNameFromDataRegion, resource, isLoaded, false);
+ new PipeTsFileInsertionEvent(isTableModel, databaseNameFromDataRegion, resource, isLoaded);
return TS_FILE_EPOCH_MANAGER.bindPipeTsFileInsertionEvent(tsFileInsertionEvent, resource);
}
@@ -52,14 +50,13 @@ public class PipeRealtimeEventFactory {
public static PipeRealtimeEvent createRealtimeEvent(
final Boolean isTableModel,
final String databaseNameFromDataRegion,
- final WALEntryHandler walEntryHandler,
final InsertNode insertNode,
final TsFileResource resource) {
final PipeInsertNodeTabletInsertionEvent insertionEvent =
new PipeInsertNodeTabletInsertionEvent(
isTableModel,
databaseNameFromDataRegion,
- walEntryHandler,
+ insertNode,
insertNode.getTargetPath(),
(insertNode instanceof InsertRowsNode)
? ((InsertRowsNode) insertNode)
@@ -70,10 +67,7 @@ public class PipeRealtimeEventFactory {
.getDeviceID(node.getTargetPath())
.getTableName())
.collect(Collectors.toSet())
- : null,
- insertNode.getProgressIndex(),
- insertNode.isAligned(),
- insertNode.isGeneratedByPipe());
+ : null);
return TS_FILE_EPOCH_MANAGER.bindPipeInsertNodeTabletInsertionEvent(
insertionEvent, insertNode, resource);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
index a8d95d44810..612b1a9701f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/DataRegionWatermarkInjector.java
@@ -48,10 +48,6 @@ public class DataRegionWatermarkInjector {
return injectionIntervalInMs;
}
- public long getNextInjectionTime() {
- return nextInjectionTime;
- }
-
public PipeWatermarkEvent inject() {
if (System.currentTimeMillis() < nextInjectionTime) {
return null;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
index 7154d901748..8b46c1e31f7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.pipe.extractor.dataregion;
import org.apache.iotdb.commons.consensus.DataRegionId;
-import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern;
@@ -41,7 +40,6 @@ import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode;
import org.apache.iotdb.pipe.api.annotation.TableModel;
import org.apache.iotdb.pipe.api.annotation.TreeModel;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
@@ -53,11 +51,12 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.apache.iotdb.pipe.api.exception.PipeException;
import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
-import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
+
import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicReference;
@@ -133,13 +132,12 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionExtractor.class);
- private PipeHistoricalDataRegionExtractor historicalExtractor;
+ private @Nullable PipeHistoricalDataRegionExtractor historicalExtractor;
private PipeRealtimeDataRegionExtractor realtimeExtractor;
private DataRegionWatermarkInjector watermarkInjector;
private boolean hasNoExtractionNeed = true;
- private boolean shouldExtractInsertion = false;
private boolean shouldExtractDeletion = false;
@Override
@@ -216,7 +214,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
return;
}
hasNoExtractionNeed = false;
- shouldExtractInsertion = insertionDeletionListeningOptionPair.getLeft();
shouldExtractDeletion = insertionDeletionListeningOptionPair.getRight();
if (insertionDeletionListeningOptionPair.getLeft().equals(true)
@@ -300,10 +297,22 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
checkInvalidParameters(validator);
- constructHistoricalExtractor();
+ if (validator
+ .getParameters()
+ .getBooleanOrDefault(SystemConstant.RESTART_KEY, SystemConstant.RESTART_DEFAULT_VALUE)
+ || validator
+ .getParameters()
+ .getBooleanOrDefault(
+ Arrays.asList(EXTRACTOR_HISTORY_ENABLE_KEY, SOURCE_HISTORY_ENABLE_KEY),
+ EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE)) {
+ // Do not flush or open historical extractor when historical tsFile is disabled
+ constructHistoricalExtractor();
+ }
constructRealtimeExtractor(validator.getParameters());
- historicalExtractor.validate(validator);
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.validate(validator);
+ }
realtimeExtractor.validate(validator);
}
@@ -430,12 +439,10 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
private void constructHistoricalExtractor() {
- // Enable historical extractor by default
historicalExtractor = new PipeHistoricalDataRegionTsFileAndDeletionExtractor();
}
- private void constructRealtimeExtractor(final PipeParameters parameters)
- throws IllegalPathException {
+ private void constructRealtimeExtractor(final PipeParameters parameters) {
// Use heartbeat only extractor if disable realtime extractor
if (!parameters.getBooleanOrDefault(
Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
@@ -473,7 +480,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
// Use hybrid mode by default
if (!parameters.hasAnyAttributes(EXTRACTOR_MODE_STREAMING_KEY, SOURCE_MODE_STREAMING_KEY)
&& !parameters.hasAnyAttributes(EXTRACTOR_REALTIME_MODE_KEY, SOURCE_REALTIME_MODE_KEY)) {
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
LOGGER.info(
"Pipe: '{}' ('{}') and '{}' ('{}') is not set, use hybrid mode by default.",
@@ -490,7 +496,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
Arrays.asList(EXTRACTOR_MODE_STREAMING_KEY, SOURCE_MODE_STREAMING_KEY),
EXTRACTOR_MODE_STREAMING_DEFAULT_VALUE);
if (isStreamingMode) {
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
} else {
realtimeExtractor = new PipeRealtimeDataRegionTsFileExtractor();
@@ -506,15 +511,12 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
case EXTRACTOR_REALTIME_MODE_HYBRID_VALUE:
case EXTRACTOR_REALTIME_MODE_LOG_VALUE:
case EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
break;
case EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionLogExtractor();
break;
default:
- checkWalEnableAndSetUncompressed(parameters);
realtimeExtractor = new PipeRealtimeDataRegionHybridExtractor();
if (LOGGER.isWarnEnabled()) {
LOGGER.warn(
@@ -524,28 +526,6 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
}
- private void checkWalEnableAndSetUncompressed(final PipeParameters parameters)
- throws IllegalPathException {
- if (Boolean.TRUE.equals(
- DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters)
- .getLeft())
- && IoTDBDescriptor.getInstance().getConfig().getWalMode().equals(WALMode.DISABLE)) {
- throw new PipeException(
- "The pipe cannot transfer realtime insertion if data region disables wal. Please set 'realtime.mode'='batch' in source parameters when enabling realtime transmission.");
- }
-
- if (!IoTDBDescriptor.getInstance()
- .getConfig()
- .getWALCompressionAlgorithm()
- .equals(CompressionType.UNCOMPRESSED)) {
- LOGGER.info(
- "The pipe prefers uncompressed wal, and may introduce certain delay in realtime insert syncing without it. Hence, we change it to uncompressed.");
- IoTDBDescriptor.getInstance()
- .getConfig()
- .setWALCompressionAlgorithm(CompressionType.UNCOMPRESSED);
- }
- }
-
@Override
public void customize(
final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration)
@@ -556,7 +536,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
super.customize(parameters, configuration);
- historicalExtractor.customize(parameters, configuration);
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.customize(parameters, configuration);
+ }
realtimeExtractor.customize(parameters, configuration);
// Set watermark injector
@@ -600,7 +582,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
"Pipe {}@{}: Starting historical extractor {} and realtime extractor {}.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName());
super.start();
@@ -635,7 +619,9 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
"Pipe {}@{}: Started historical extractor {} and realtime extractor {} successfully within {} ms.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName(),
System.currentTimeMillis() - startTime);
return;
@@ -653,14 +639,18 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
// There can still be writing when tsFile events are added. If we start
// realtimeExtractor after the process, then this part of data will be lost.
realtimeExtractor.start();
- historicalExtractor.start();
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.start();
+ }
} catch (final Exception e) {
exceptionHolder.set(e);
LOGGER.warn(
"Pipe {}@{}: Start historical extractor {} and realtime extractor {} error.",
pipeName,
regionId,
- historicalExtractor.getClass().getSimpleName(),
+ Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getClass().getSimpleName()
+ : null,
realtimeExtractor.getClass().getSimpleName(),
e);
}
@@ -679,7 +669,7 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
}
Event event = null;
- if (!historicalExtractor.hasConsumedAll()) {
+ if (Objects.nonNull(historicalExtractor) && !historicalExtractor.hasConsumedAll()) {
event = historicalExtractor.supply();
} else {
if (Objects.nonNull(watermarkInjector)) {
@@ -709,32 +699,21 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
return;
}
- historicalExtractor.close();
+ if (Objects.nonNull(historicalExtractor)) {
+ historicalExtractor.close();
+ }
realtimeExtractor.close();
if (Objects.nonNull(taskID)) {
PipeDataRegionExtractorMetrics.getInstance().deregister(taskID);
}
}
- //////////////////////////// APIs provided for detecting stuck ////////////////////////////
-
- public boolean shouldExtractInsertion() {
- return shouldExtractInsertion;
- }
-
- public boolean isStreamMode() {
- return realtimeExtractor instanceof PipeRealtimeDataRegionHybridExtractor
- || realtimeExtractor instanceof PipeRealtimeDataRegionLogExtractor;
- }
-
- public boolean hasConsumedAllHistoricalTsFiles() {
- return historicalExtractor.hasConsumedAll();
- }
-
//////////////////////////// APIs provided for metric framework ////////////////////////////
public int getHistoricalTsFileInsertionEventCount() {
- return hasBeenStarted.get() ? historicalExtractor.getPendingQueueSize() : 0;
+ return hasBeenStarted.get() && Objects.nonNull(historicalExtractor)
+ ? historicalExtractor.getPendingQueueSize()
+ : 0;
}
public int getTabletInsertionEventCount() {
@@ -748,10 +727,4 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor {
public int getPipeHeartbeatEventCount() {
return hasBeenStarted.get() ? realtimeExtractor.getPipeHeartbeatEventCount() : 0;
}
-
- public int getEventCount() {
- return hasBeenStarted.get()
- ? (historicalExtractor.getPendingQueueSize() + realtimeExtractor.getEventCount())
- : 0;
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
index b1ebe126a3e..bdda4117432 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
@@ -49,7 +49,6 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
@@ -532,7 +531,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
List<PersistentResource> originalResourceList = new ArrayList<>();
if (shouldExtractInsertion) {
- flushTsFilesForExtraction(dataRegion, startHistoricalExtractionTime);
+ flushTsFilesForExtraction(dataRegion);
extractTsFiles(dataRegion, startHistoricalExtractionTime, originalResourceList);
}
if (shouldExtractDeletion) {
@@ -560,8 +559,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
}
}
- private void flushTsFilesForExtraction(
- DataRegion dataRegion, final long startHistoricalExtractionTime) {
+ private void flushTsFilesForExtraction(DataRegion dataRegion) {
LOGGER.info("Pipe {}@{}: start to flush data region", pipeName, dataRegionId);
// Consider the scenario: a consensus pipe comes to the same region, followed by another pipe
@@ -571,31 +569,8 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
// consensus pipe, and the lastFlushed timestamp is not updated here.
if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) {
dataRegion.syncCloseAllWorkingTsFileProcessors();
- LOGGER.info(
- "Pipe {}@{}: finish to flush data region, took {} ms",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - startHistoricalExtractionTime);
- return;
- }
-
- synchronized (DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP) {
- final long lastFlushedByPipeTime = DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.get(dataRegionId);
- if (System.currentTimeMillis() - lastFlushedByPipeTime >= PIPE_MIN_FLUSH_INTERVAL_IN_MS) {
- dataRegion.syncCloseAllWorkingTsFileProcessors();
- DATA_REGION_ID_TO_PIPE_FLUSHED_TIME_MAP.replace(dataRegionId, System.currentTimeMillis());
- LOGGER.info(
- "Pipe {}@{}: finish to flush data region, took {} ms",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - startHistoricalExtractionTime);
- } else {
- LOGGER.info(
- "Pipe {}@{}: skip to flush data region, last flushed time {} ms ago",
- pipeName,
- dataRegionId,
- System.currentTimeMillis() - lastFlushedByPipeTime);
- }
+ } else {
+ dataRegion.asyncCloseAllWorkingTsFileProcessors();
}
}
@@ -665,7 +640,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
// Will unpin it after the PipeTsFileInsertionEvent is created and pinned.
try {
PipeDataNodeResourceManager.tsfile()
- .pinTsFileResource(resource, shouldTransferModFile);
+ .pinTsFileResource(resource, shouldTransferModFile, pipeName);
return false;
} catch (final IOException e) {
LOGGER.warn("Pipe: failed to pin TsFileResource {}", resource.getTsFilePath(), e);
@@ -729,9 +704,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
try {
final Map<IDeviceID, Boolean> deviceIsAlignedMap =
PipeDataNodeResourceManager.tsfile()
- .getDeviceIsAlignedMapFromCache(
- PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile()),
- false);
+ .getDeviceIsAlignedMapFromCache(resource.getTsFile(), false);
deviceSet =
Objects.nonNull(deviceIsAlignedMap) ? deviceIsAlignedMap.keySet() : resource.getDevices();
} catch (final IOException e) {
@@ -913,6 +886,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
isModelDetected ? isTableModel : null,
resource.getDatabaseName(),
resource,
+ null,
shouldTransferModFile,
false,
true,
@@ -956,7 +930,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
return isReferenceCountIncreased ? event : null;
} finally {
try {
- PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource);
+ PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName);
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after creating event, original path: {}",
@@ -1026,7 +1000,8 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
resource -> {
if (resource instanceof TsFileResource) {
try {
- PipeDataNodeResourceManager.tsfile().unpinTsFileResource((TsFileResource) resource);
+ PipeDataNodeResourceManager.tsfile()
+ .unpinTsFileResource((TsFileResource) resource, pipeName);
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after dropping pipe, original path: {}",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
index 1b5f05fc8fb..11b4cbbbeb2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.pipe.extractor.dataregion.realtime;
import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.consensus.index.ProgressIndex;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
@@ -33,8 +34,11 @@ import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.DataRegionListeningFilter;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeTimePartitionListener;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
@@ -436,6 +440,13 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
}
}
+ protected void maySkipIndex4Event(final PipeRealtimeEvent event) {
+ if (event.getEvent() instanceof PipeTsFileInsertionEvent
+ || event.getEvent() instanceof PipeInsertNodeTabletInsertionEvent) {
+ maySkipProgressIndexForRealtimeEvent(event);
+ }
+ }
+
protected Event supplyHeartbeat(final PipeRealtimeEvent event) {
if (event.increaseReferenceCount(PipeRealtimeDataRegionExtractor.class.getName())) {
return event.getEvent();
@@ -549,6 +560,30 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
return shouldTransferModFile;
}
+ private void maySkipProgressIndexForRealtimeEvent(final PipeRealtimeEvent event) {
+ if (PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .isProgressIndexAfterOrEquals(
+ dataRegionId,
+ pipeName,
+ event.getTsFileEpoch().getFilePath(),
+ getProgressIndex4RealtimeEvent(event))) {
+ event.skipReportOnCommit();
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Pipe {} on data region {} skip commit of event {} because it was flushed prematurely.",
+ pipeName,
+ dataRegionId,
+ event.coreReportMessage());
+ }
+ }
+ }
+
+ private ProgressIndex getProgressIndex4RealtimeEvent(final PipeRealtimeEvent event) {
+ return event.getEvent() instanceof PipeTsFileInsertionEvent
+ ? ((PipeTsFileInsertionEvent) event.getEvent()).forceGetProgressIndex()
+ : event.getProgressIndex();
+ }
+
@Override
public String toString() {
return toStringHelper(this)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
index ae632237650..8ee2f7a15a1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime;
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
@@ -31,11 +30,8 @@ import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch.TsFileEpoch;
-import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.storageengine.StorageEngine;
-import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
@@ -44,7 +40,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Objects;
-import java.util.function.Consumer;
public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegionExtractor {
@@ -86,6 +81,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
if (canNotUseTabletAnyMore(event)) {
event.getTsFileEpoch().migrateState(this, curState -> TsFileEpoch.State.USING_TSFILE);
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .registerProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getResource());
} else {
event
.getTsFileEpoch()
@@ -144,6 +141,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
}
private void extractTsFileInsertion(final PipeRealtimeEvent event) {
+ // Notice that, if the tsFile is partially extracted because the pipe is not opened before, the
+ // former data won't be extracted
event
.getTsFileEpoch()
.migrateState(
@@ -151,34 +150,13 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
state -> {
switch (state) {
case EMPTY:
+ return ((PipeTsFileInsertionEvent) event.getEvent()).isLoaded()
+ ? TsFileEpoch.State.USING_TSFILE
+ : TsFileEpoch.State.USING_TABLET;
+ case USING_TABLET:
+ return TsFileEpoch.State.USING_TABLET;
case USING_TSFILE:
return TsFileEpoch.State.USING_TSFILE;
- case USING_TABLET:
- if (((PipeTsFileInsertionEvent) event.getEvent()).getFileStartTime()
- < event.getTsFileEpoch().getInsertNodeMinTime()) {
- // Some insert nodes in the tsfile epoch are not captured by pipe, so we should
- // capture the tsfile event to make sure all data in the tsfile epoch can be
- // extracted.
- //
- // The situation can be caused by the following operations:
- // 1. PipeA: start historical data extraction with flush
- // 2. Data insertion
- // 3. PipeB: start realtime data extraction
- // 4. PipeB: start historical data extraction without flush
- // 5. Data inserted in the step2 is not captured by PipeB, and if its tsfile
- // epoch's state is USING_TABLET, the tsfile event will be ignored, which
- // will cause the data loss in the tsfile epoch.
- LOGGER.info(
- "The tsFile {}'s epoch's start time {} is smaller than the captured insertNodes' min time {}, will regard it as data loss or un-sequential, will extract the tsFile",
- ((PipeTsFileInsertionEvent) event.getEvent()).getTsFile(),
- ((PipeTsFileInsertionEvent) event.getEvent()).getFileStartTime(),
- event.getTsFileEpoch().getInsertNodeMinTime());
- return TsFileEpoch.State.USING_BOTH;
- } else {
- // All data in the tsfile epoch has been extracted in tablet mode, so we should
- // simply keep the state of the tsfile epoch and discard the tsfile event.
- return TsFileEpoch.State.USING_TABLET;
- }
case USING_BOTH:
default:
return canNotUseTabletAnyMore(event)
@@ -224,50 +202,13 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
private boolean canNotUseTabletAnyMore(final PipeRealtimeEvent event) {
// In the following 4 cases, we should not extract this tablet event. all the data
// represented by the tablet event should be carried by the following tsfile event:
- // 0. If the remaining insert event count is too large, we need to reduce the accumulated
- // tablets.
- // 1. If Wal size > maximum size of wal buffer,
// the write operation will be throttled, so we should not extract any more tablet events.
- // 2. The shallow memory usage of the insert node has reached the dangerous threshold.
- // 3. Deprecated logics (unused by default)
- return mayRemainingInsertNodeEventExceedLimit(event)
- || mayWalSizeReachThrottleThreshold(event)
- || mayInsertNodeMemoryReachDangerousThreshold(event)
+ // 1. The shallow memory usage of the insert node has reached the dangerous threshold.
+ // 2. Deprecated logics (unused by default)
+ return mayInsertNodeMemoryReachDangerousThreshold(event)
|| canNotUseTabletAnymoreDeprecated(event);
}
- private boolean mayRemainingInsertNodeEventExceedLimit(final PipeRealtimeEvent event) {
- final boolean mayRemainingInsertEventExceedLimit =
- PipeDataNodeSinglePipeMetrics.getInstance().mayRemainingInsertEventExceedLimit(pipeID);
- if (mayRemainingInsertEventExceedLimit && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(0): remaining insert event has reached max allowed insert event count {}",
- pipeName,
- dataRegionId,
- PipeConfig.getInstance().getPipeMaxAllowedRemainingInsertEventCountPerPipe()));
- }
- return mayRemainingInsertEventExceedLimit;
- }
-
- private boolean mayWalSizeReachThrottleThreshold(final PipeRealtimeEvent event) {
- final boolean mayWalSizeReachThrottleThreshold =
- 3 * WALManager.getInstance().getTotalDiskUsage()
- > IoTDBDescriptor.getInstance().getConfig().getThrottleThreshold();
- if (mayWalSizeReachThrottleThreshold && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(1): Wal size {} has reached throttle threshold {}",
- pipeName,
- dataRegionId,
- WALManager.getInstance().getTotalDiskUsage(),
- IoTDBDescriptor.getInstance().getConfig().getThrottleThreshold() / 3.0d));
- }
- return mayWalSizeReachThrottleThreshold;
- }
-
private boolean mayInsertNodeMemoryReachDangerousThreshold(final PipeRealtimeEvent event) {
final long floatingMemoryUsageInByte =
PipeDataNodeAgent.task().getFloatingMemoryUsageInByte(pipeName);
@@ -275,27 +216,19 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
final long totalFloatingMemorySizeInBytes =
PipeDataNodeResourceManager.memory().getTotalFloatingMemorySizeInBytes();
final boolean mayInsertNodeMemoryReachDangerousThreshold =
- 3 * floatingMemoryUsageInByte * pipeCount >= 2 * totalFloatingMemorySizeInBytes;
+ floatingMemoryUsageInByte * pipeCount >= totalFloatingMemorySizeInBytes;
if (mayInsertNodeMemoryReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
- logByLogManager(
- l ->
- l.info(
- "Pipe task {}@{} canNotUseTabletAnyMore(2): The shallow memory usage of the insert node {} has reached the dangerous threshold {}",
- pipeName,
- dataRegionId,
- floatingMemoryUsageInByte * pipeCount,
- 2 * totalFloatingMemorySizeInBytes / 3.0d));
+ LOGGER.info(
+ "Pipe task {}@{} canNotUseTabletAnyMore(1) for tsFile {}: The memory usage of the insert node {} has reached the dangerous threshold {}",
+ pipeName,
+ dataRegionId,
+ event.getTsFileEpoch().getFilePath(),
+ floatingMemoryUsageInByte * pipeCount,
+ totalFloatingMemorySizeInBytes);
}
return mayInsertNodeMemoryReachDangerousThreshold;
}
- private void logByLogManager(final Consumer<Logger> infoFunction) {
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeRealtimeDataRegionHybridExtractor.class, getTaskID(), Integer.MAX_VALUE, 100, 1)
- .ifPresent(infoFunction);
- }
-
/**
* These judgements are deprecated, and are only reserved for manual operation and compatibility.
*/
@@ -303,54 +236,14 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
private boolean canNotUseTabletAnymoreDeprecated(final PipeRealtimeEvent event) {
// In the following 5 cases, we should not extract any more tablet events. all the data
// represented by the tablet events should be carried by the following tsfile event:
- // 0. If the pipe task is currently restarted.
- // 1. The number of pinned memTables has reached the dangerous threshold.
- // 2. The number of historical tsFile events to transfer has exceeded the limit.
- // 3. The number of realtime tsfile events to transfer has exceeded the limit.
- // 4. The number of linked tsFiles has reached the dangerous threshold.
- return isPipeTaskCurrentlyRestarted(event)
- || mayMemTablePinnedCountReachDangerousThreshold(event)
- || isHistoricalTsFileEventCountExceededLimit(event)
+ // 1. The number of historical tsFile events to transfer has exceeded the limit.
+ // 2. The number of realtime tsfile events to transfer has exceeded the limit.
+ // 3. The number of linked tsFiles has reached the dangerous threshold.
+ return isHistoricalTsFileEventCountExceededLimit(event)
|| isRealtimeTsFileEventCountExceededLimit(event)
|| mayTsFileLinkedCountReachDangerousThreshold(event);
}
- private boolean isPipeTaskCurrentlyRestarted(final PipeRealtimeEvent event) {
- if (!PipeConfig.getInstance().isPipeEpochKeepTsFileAfterStuckRestartEnabled()) {
- return false;
- }
-
- final boolean isPipeTaskCurrentlyRestarted =
- PipeDataNodeAgent.task().isPipeTaskCurrentlyRestarted(pipeName);
- if (isPipeTaskCurrentlyRestarted && event.mayExtractorUseTablets(this)) {
- LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(0): Pipe task is currently restarted",
- pipeName,
- dataRegionId);
- }
- return isPipeTaskCurrentlyRestarted;
- }
-
- private boolean mayMemTablePinnedCountReachDangerousThreshold(final PipeRealtimeEvent event) {
- if (PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount() == Integer.MAX_VALUE) {
- return false;
- }
- final boolean mayMemTablePinnedCountReachDangerousThreshold =
- PipeDataNodeResourceManager.wal().getPinnedWalCount()
- >= PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber();
- if (mayMemTablePinnedCountReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
- LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(1): The number of pinned memTables {} has reached the dangerous threshold {}",
- pipeName,
- dataRegionId,
- PipeDataNodeResourceManager.wal().getPinnedWalCount(),
- PipeConfig.getInstance().getPipeMaxAllowedPinnedMemTableCount()
- * StorageEngine.getInstance().getDataRegionNumber());
- }
- return mayMemTablePinnedCountReachDangerousThreshold;
- }
-
private boolean isHistoricalTsFileEventCountExceededLimit(final PipeRealtimeEvent event) {
if (PipeConfig.getInstance().getPipeMaxAllowedHistoricalTsFilePerDataRegion()
== Integer.MAX_VALUE) {
@@ -364,9 +257,10 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
>= PipeConfig.getInstance().getPipeMaxAllowedHistoricalTsFilePerDataRegion();
if (isHistoricalTsFileEventCountExceededLimit && event.mayExtractorUseTablets(this)) {
LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(2): The number of historical tsFile events {} has exceeded the limit {}",
+ "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(1) for tsFile {}: The number of historical tsFile events {} has exceeded the limit {}",
pipeName,
dataRegionId,
+ event.getTsFileEpoch().getFilePath(),
extractor.getHistoricalTsFileInsertionEventCount(),
PipeConfig.getInstance().getPipeMaxAllowedHistoricalTsFilePerDataRegion());
}
@@ -383,9 +277,10 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
>= PipeConfig.getInstance().getPipeMaxAllowedPendingTsFileEpochPerDataRegion();
if (isRealtimeTsFileEventCountExceededLimit && event.mayExtractorUseTablets(this)) {
LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(3): The number of realtime tsFile events {} has exceeded the limit {}",
+ "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(2) for tsFile {}: The number of realtime tsFile events {} has exceeded the limit {}",
pipeName,
dataRegionId,
+ event.getTsFileEpoch().getFilePath(),
pendingQueue.getTsFileInsertionEventCount(),
PipeConfig.getInstance().getPipeMaxAllowedPendingTsFileEpochPerDataRegion());
}
@@ -397,14 +292,15 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
return false;
}
final boolean mayTsFileLinkedCountReachDangerousThreshold =
- PipeDataNodeResourceManager.tsfile().getLinkedTsfileCount()
+ PipeDataNodeResourceManager.tsfile().getLinkedTsFileCount(pipeName)
>= PipeConfig.getInstance().getPipeMaxAllowedLinkedTsFileCount();
if (mayTsFileLinkedCountReachDangerousThreshold && event.mayExtractorUseTablets(this)) {
LOGGER.info(
- "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(4): The number of linked tsFiles {} has reached the dangerous threshold {}",
+ "Pipe task {}@{} canNotUseTabletAnymoreDeprecated(3) for tsFile {}: The number of linked tsFiles {} has reached the dangerous threshold {}",
pipeName,
dataRegionId,
- PipeDataNodeResourceManager.tsfile().getLinkedTsfileCount(),
+ event.getTsFileEpoch().getFilePath(),
+ PipeDataNodeResourceManager.tsfile().getLinkedTsFileCount(pipeName),
PipeConfig.getInstance().getPipeMaxAllowedLinkedTsFileCount());
}
return mayTsFileLinkedCountReachDangerousThreshold;
@@ -439,6 +335,7 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
PipeRealtimeDataRegionHybridExtractor.class.getName(), false);
if (suppliedEvent != null) {
+ maySkipIndex4Event(realtimeEvent);
return suppliedEvent;
}
@@ -472,11 +369,14 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
});
final TsFileEpoch.State state = event.getTsFileEpoch().getState(this);
+ if (state == TsFileEpoch.State.USING_TSFILE) {
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .registerProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getResource());
+ }
+
switch (state) {
case USING_TSFILE:
// If the state is USING_TSFILE, discard the event and poll the next one.
- PipeTsFileEpochProgressIndexKeeper.getInstance()
- .eliminateProgressIndex(dataRegionId, event.getTsFileEpoch().getFilePath());
return null;
case EMPTY:
case USING_TABLET:
@@ -517,6 +417,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
switch (state) {
case USING_TABLET:
// If the state is USING_TABLET, discard the event and poll the next one.
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getFilePath());
return null;
case EMPTY:
case USING_TSFILE:
@@ -538,6 +440,8 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
LOGGER.error(errorMessage);
PipeDataNodeAgent.runtime()
.report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getFilePath());
return null;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
index 1348bfdbaa9..3d2a39128cf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionLogExtractor.java
@@ -82,10 +82,7 @@ public class PipeRealtimeDataRegionLogExtractor extends PipeRealtimeDataRegionEx
private void extractTsFileInsertion(PipeRealtimeEvent event) {
final PipeTsFileInsertionEvent tsFileInsertionEvent =
(PipeTsFileInsertionEvent) event.getEvent();
- if (!(tsFileInsertionEvent.isLoaded()
- // some insert nodes in the tsfile epoch are not captured by pipe
- || tsFileInsertionEvent.getFileStartTime()
- < event.getTsFileEpoch().getInsertNodeMinTime())) {
+ if (!(tsFileInsertionEvent.isLoaded())) {
// All data in the tsfile epoch has been extracted in tablet mode, so we should
// simply ignore this event.
event.decreaseReferenceCount(PipeRealtimeDataRegionLogExtractor.class.getName(), false);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
index 27198e349cb..aafb2789ea4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionTsFileExtractor.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
+import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch.TsFileEpoch;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
@@ -50,6 +51,8 @@ public class PipeRealtimeDataRegionTsFileExtractor extends PipeRealtimeDataRegio
}
event.getTsFileEpoch().migrateState(this, state -> TsFileEpoch.State.USING_TSFILE);
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .registerProgressIndex(dataRegionId, pipeName, event.getTsFileEpoch().getResource());
if (!(event.getEvent() instanceof TsFileInsertionEvent)) {
event.decreaseReferenceCount(PipeRealtimeDataRegionTsFileExtractor.class.getName(), false);
@@ -111,12 +114,16 @@ public class PipeRealtimeDataRegionTsFileExtractor extends PipeRealtimeDataRegio
LOGGER.error(errorMessage);
PipeDataNodeAgent.runtime()
.report(pipeTaskMeta, new PipeRuntimeNonCriticalException(errorMessage));
+ PipeTsFileEpochProgressIndexKeeper.getInstance()
+ .eliminateProgressIndex(
+ dataRegionId, pipeName, realtimeEvent.getTsFileEpoch().getFilePath());
}
realtimeEvent.decreaseReferenceCount(
PipeRealtimeDataRegionTsFileExtractor.class.getName(), false);
if (suppliedEvent != null) {
+ maySkipIndex4Event(realtimeEvent);
return suppliedEvent;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
index 060b395e51a..6238117ed89 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
@@ -34,7 +34,6 @@ import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource;
import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager;
import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent;
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
-import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEventFactory;
@@ -46,7 +45,6 @@ import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
-import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,8 +70,6 @@ public class PipeDataRegionAssigner implements Closeable {
private final String dataRegionId;
- private int counter = 0;
-
private Boolean isTableModel;
private final AtomicReference<ProgressIndex> maxProgressIndexForRealtimeEvent =
@@ -130,10 +126,11 @@ public class PipeDataRegionAssigner implements Closeable {
realtimeEvent.decreaseReferenceCount(PipeDataRegionAssigner.class.getName(), false);
final EnrichedEvent innerEvent = realtimeEvent.getEvent();
- eventCounter.decreaseEventCount(innerEvent);
if (innerEvent instanceof PipeHeartbeatEvent) {
((PipeHeartbeatEvent) innerEvent).onAssigned();
}
+
+ eventCounter.decreaseEventCount(innerEvent);
}
private void assignToExtractor(
@@ -151,16 +148,6 @@ public class PipeDataRegionAssigner implements Closeable {
}
if (event.getEvent().isGeneratedByPipe() && !extractor.isForwardingPipeRequests()) {
- // The frequency of progress reports is limited by the counter, while progress
- // reports to TsFileInsertionEvent are not limited.
- if (!(event.getEvent() instanceof TsFileInsertionEvent)) {
- if (counter < PIPE_CONFIG.getPipeNonForwardingEventsProgressReportInterval()) {
- counter++;
- return;
- }
- counter = 0;
- }
-
final ProgressReportEvent reportEvent =
new ProgressReportEvent(
extractor.getPipeName(),
@@ -215,11 +202,6 @@ public class PipeDataRegionAssigner implements Closeable {
extractor.isShouldTransferModFile());
}
- if (innerEvent instanceof PipeTsFileInsertionEvent
- || innerEvent instanceof PipeInsertNodeTabletInsertionEvent) {
- bindOrUpdateProgressIndexForRealtimeEvent(copiedEvent);
- }
-
if (innerEvent instanceof PipeDeleteDataNodeEvent) {
final PipeDeleteDataNodeEvent deleteDataNodeEvent =
(PipeDeleteDataNodeEvent) innerEvent;
@@ -245,34 +227,6 @@ public class PipeDataRegionAssigner implements Closeable {
});
}
- private void bindOrUpdateProgressIndexForRealtimeEvent(final PipeRealtimeEvent event) {
- if (PipeTsFileEpochProgressIndexKeeper.getInstance()
- .isProgressIndexAfterOrEquals(
- dataRegionId,
- event.getTsFileEpoch().getFilePath(),
- getProgressIndex4RealtimeEvent(event))) {
- event.bindProgressIndex(maxProgressIndexForRealtimeEvent.get());
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(
- "Data region {} bind {} to event {} because it was flushed prematurely.",
- dataRegionId,
- maxProgressIndexForRealtimeEvent,
- event.coreReportMessage());
- }
- } else {
- maxProgressIndexForRealtimeEvent.updateAndGet(
- index ->
- index.updateToMinimumEqualOrIsAfterProgressIndex(
- getProgressIndex4RealtimeEvent(event)));
- }
- }
-
- private ProgressIndex getProgressIndex4RealtimeEvent(final PipeRealtimeEvent event) {
- return event.getEvent() instanceof PipeTsFileInsertionEvent
- ? ((PipeTsFileInsertionEvent) event.getEvent()).forceGetProgressIndex()
- : event.getProgressIndex();
- }
-
public void startAssignTo(final PipeRealtimeDataRegionExtractor extractor) {
matcher.register(extractor);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java
index da2cde90667..50b7d778fc2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeTsFileEpochProgressIndexKeeper.java
@@ -20,6 +20,9 @@
package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner;
import org.apache.iotdb.commons.consensus.index.ProgressIndex;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import javax.annotation.Nonnull;
import java.util.Map;
import java.util.Map.Entry;
@@ -28,39 +31,40 @@ import java.util.concurrent.ConcurrentHashMap;
public class PipeTsFileEpochProgressIndexKeeper {
- // data region id -> (tsFile path, max progress index)
- private final Map<String, Map<String, ProgressIndex>> progressIndexKeeper =
+ // data region id -> pipeName -> tsFile path -> max progress index
+ private final Map<String, Map<String, Map<String, TsFileResource>>> progressIndexKeeper =
new ConcurrentHashMap<>();
- public synchronized void updateProgressIndex(
- final String dataRegionId, final String tsFileName, final ProgressIndex progressIndex) {
+ public synchronized void registerProgressIndex(
+ final String dataRegionId, final String pipeName, final TsFileResource resource) {
progressIndexKeeper
.computeIfAbsent(dataRegionId, k -> new ConcurrentHashMap<>())
- .compute(
- tsFileName,
- (k, v) ->
- v == null
- ? progressIndex
- : v.updateToMinimumEqualOrIsAfterProgressIndex(progressIndex));
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
+ .putIfAbsent(resource.getTsFilePath(), resource);
}
public synchronized void eliminateProgressIndex(
- final String dataRegionId, final String filePath) {
+ final String dataRegionId, final @Nonnull String pipeName, final String filePath) {
progressIndexKeeper
.computeIfAbsent(dataRegionId, k -> new ConcurrentHashMap<>())
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
.remove(filePath);
}
public synchronized boolean isProgressIndexAfterOrEquals(
- final String dataRegionId, final String tsFilePath, final ProgressIndex progressIndex) {
+ final String dataRegionId,
+ final String pipeName,
+ final String tsFilePath,
+ final ProgressIndex progressIndex) {
return progressIndexKeeper
.computeIfAbsent(dataRegionId, k -> new ConcurrentHashMap<>())
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
.entrySet()
.stream()
.filter(entry -> !Objects.equals(entry.getKey(), tsFilePath))
.map(Entry::getValue)
.filter(Objects::nonNull)
- .anyMatch(index -> !index.isAfter(progressIndex));
+ .anyMatch(resource -> !resource.getMaxProgressIndex().isAfter(progressIndex));
}
//////////////////////////// singleton ////////////////////////////
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java
index c2db4c77c86..83b69ea60ec 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpoch.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.pipe.extractor.dataregion.realtime.epoch;
import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.PipeRealtimeDataRegionExtractor;
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionExtractorMetrics;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -29,13 +30,13 @@ import java.util.concurrent.atomic.AtomicReference;
public class TsFileEpoch {
- private final String filePath;
+ private final TsFileResource resource;
private final ConcurrentMap<PipeRealtimeDataRegionExtractor, AtomicReference<State>>
dataRegionExtractor2State;
private final AtomicLong insertNodeMinTime;
- public TsFileEpoch(final String filePath) {
- this.filePath = filePath;
+ public TsFileEpoch(final TsFileResource resource) {
+ this.resource = resource;
this.dataRegionExtractor2State = new ConcurrentHashMap<>();
this.insertNodeMinTime = new AtomicLong(Long.MAX_VALUE);
}
@@ -64,19 +65,19 @@ public class TsFileEpoch {
insertNodeMinTime.updateAndGet(recordedMinTime -> Math.min(recordedMinTime, newComingMinTime));
}
- public long getInsertNodeMinTime() {
- return insertNodeMinTime.get();
+ public TsFileResource getResource() {
+ return resource;
}
public String getFilePath() {
- return filePath;
+ return resource.getTsFilePath();
}
@Override
public String toString() {
return "TsFileEpoch{"
- + "filePath='"
- + filePath
+ + "resource='"
+ + resource
+ '\''
+ ", dataRegionExtractor2State="
+ dataRegionExtractor2State
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java
index 869f9244e87..8613c611536 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/epoch/TsFileEpochManager.java
@@ -57,7 +57,7 @@ public class TsFileEpochManager {
filePath,
path -> {
LOGGER.info("TsFileEpoch not found for TsFile {}, creating a new one", path);
- return new TsFileEpoch(path);
+ return new TsFileEpoch(resource);
});
final TsFileEpoch epoch = filePath2Epoch.remove(filePath);
@@ -76,7 +76,7 @@ public class TsFileEpochManager {
public PipeRealtimeEvent bindPipeInsertNodeTabletInsertionEvent(
PipeInsertNodeTabletInsertionEvent event, InsertNode node, TsFileResource resource) {
final TsFileEpoch epoch =
- filePath2Epoch.computeIfAbsent(resource.getTsFilePath(), TsFileEpoch::new);
+ filePath2Epoch.computeIfAbsent(resource.getTsFilePath(), k -> new TsFileEpoch(resource));
epoch.updateInsertNodeMinTime(node.getMinTime());
return new PipeRealtimeEvent(
event,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
index 90d378ecf7d..e55128a8731 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
@@ -29,7 +29,6 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeDataR
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
@@ -100,9 +99,7 @@ public class PipeInsertionDataNodeListener {
final String dataRegionId,
final String databaseName,
final TsFileResource tsFileResource,
- final boolean isLoaded,
- final boolean isGeneratedByPipe) {
- tsFileResource.setGeneratedByPipe(isGeneratedByPipe);
+ final boolean isLoaded) {
// We don't judge whether listenToTsFileExtractorCount.get() == 0 here on purpose
// because extractors may use tsfile events when some exceptions occur in the
// insert nodes listening process.
@@ -122,7 +119,6 @@ public class PipeInsertionDataNodeListener {
public void listenToInsertNode(
final String dataRegionId,
final String databaseName,
- final WALEntryHandler walEntryHandler,
final InsertNode insertNode,
final TsFileResource tsFileResource) {
if (listenToInsertNodeExtractorCount.get() == 0) {
@@ -138,7 +134,7 @@ public class PipeInsertionDataNodeListener {
assigner.publishToAssign(
PipeRealtimeEventFactory.createRealtimeEvent(
- assigner.isTableModel(), databaseName, walEntryHandler, insertNode, tsFileResource));
+ assigner.isTableModel(), databaseName, insertNode, tsFileResource));
}
public DeletionResource listenToDeleteData(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
index dd3873feaf4..d7f1577a85c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeMetrics.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeHeartbeatEventMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics;
import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics;
-import org.apache.iotdb.db.pipe.metric.overview.PipeWALInsertNodeCacheMetrics;
import org.apache.iotdb.db.pipe.metric.processor.PipeProcessorMetrics;
import org.apache.iotdb.db.pipe.metric.receiver.PipeDataNodeReceiverMetrics;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionConnectorMetrics;
@@ -47,7 +46,6 @@ public class PipeDataNodeMetrics implements IMetricSet {
PipeProcessorMetrics.getInstance().bindTo(metricService);
PipeDataRegionConnectorMetrics.getInstance().bindTo(metricService);
PipeHeartbeatEventMetrics.getInstance().bindTo(metricService);
- PipeWALInsertNodeCacheMetrics.getInstance().bindTo(metricService);
PipeResourceMetrics.getInstance().bindTo(metricService);
PipeEventCommitMetrics.getInstance().bindTo(metricService);
PipeSchemaRegionListenerMetrics.getInstance().bindTo(metricService);
@@ -65,7 +63,6 @@ public class PipeDataNodeMetrics implements IMetricSet {
PipeProcessorMetrics.getInstance().unbindFrom(metricService);
PipeDataRegionConnectorMetrics.getInstance().unbindFrom(metricService);
PipeHeartbeatEventMetrics.getInstance().unbindFrom(metricService);
- PipeWALInsertNodeCacheMetrics.getInstance().unbindFrom(metricService);
PipeResourceMetrics.getInstance().unbindFrom(metricService);
PipeEventCommitMetrics.getInstance().unbindFrom(metricService);
PipeSchemaRegionListenerMetrics.getInstance().unbindFrom(metricService);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
index 73d58285345..0308e9b5b63 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
@@ -59,8 +59,6 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
private Timer insertNodeTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer tsfileTransferTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- private final InsertNodeEMA insertNodeEventCountEMA = new InsertNodeEMA();
-
private double lastDataRegionCommitSmoothingValue = Long.MAX_VALUE;
private double lastSchemaRegionCommitSmoothingValue = Long.MAX_VALUE;
@@ -102,11 +100,6 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
heartbeatEventCount.decrementAndGet();
}
- double getRemainingInsertEventSmoothingCount() {
- insertNodeEventCountEMA.update(insertNodeEventCount.get());
- return insertNodeEventCountEMA.insertNodeEMAValue;
- }
-
public long getRemainingNonHeartbeatEvents() {
final long remainingEvents =
tsfileEventCount.get()
@@ -147,7 +140,7 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
*
* @return The estimated remaining time
*/
- double getRemainingTime() {
+ public double getRemainingTime() {
final PipeRateAverage pipeRemainingTimeCommitRateAverageTime =
PipeConfig.getInstance().getPipeRemainingTimeCommitRateAverageTime();
@@ -282,17 +275,4 @@ public class PipeDataNodeRemainingEventAndTimeOperator extends PipeRemainingOper
dataRegionCommitMeter.set(null);
schemaRegionCommitMeter.set(null);
}
-
- private static class InsertNodeEMA {
- private double insertNodeEMAValue;
-
- public void update(final double newValue) {
- final double alpha = PipeConfig.getInstance().getPipeRemainingInsertNodeCountEMAAlpha();
- if (insertNodeEMAValue == 0) {
- insertNodeEMAValue = newValue;
- } else {
- insertNodeEMAValue = alpha * newValue + (1 - alpha) * insertNodeEMAValue;
- }
- }
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
index 677d758a162..1840093a347 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
@@ -20,11 +20,12 @@
package org.apache.iotdb.db.pipe.metric.overview;
import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
@@ -54,7 +55,7 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
private static Histogram PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
- private static Histogram PIPE_DATANODE_EVENT_TRANSFER_TIME_HISTOGRAM =
+ private static Histogram PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
//////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
@@ -68,7 +69,7 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
MetricLevel.IMPORTANT,
Tag.NAME.toString(),
"insert_node");
- PIPE_DATANODE_EVENT_TRANSFER_TIME_HISTOGRAM =
+ PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM =
metricService.getOrCreateHistogram(
Metric.PIPE_DATANODE_EVENT_TRANSFER.toString(),
MetricLevel.IMPORTANT,
@@ -103,6 +104,35 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
Tag.CREATION_TIME.toString(),
String.valueOf(operator.getCreationTime()));
+ // Resources
+ metricService.createAutoGauge(
+ Metric.PIPE_FLOATING_MEMORY_USAGE.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeAgent.task(),
+ a -> a.getFloatingMemoryUsageInByte(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.createAutoGauge(
+ Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeResourceManager.tsfile(),
+ a -> a.getLinkedTsFileCount(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.createAutoGauge(
+ Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ MetricLevel.IMPORTANT,
+ PipeDataNodeResourceManager.tsfile(),
+ a -> a.getTotalLinkedTsFileSize(operator.getPipeName()),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+
operator.setInsertNodeTransferTimer(
metricService.getOrCreateTimer(
Metric.PIPE_INSERT_NODE_EVENT_TRANSFER_TIME.toString(),
@@ -118,31 +148,6 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
operator.getPipeName()));
}
- public boolean mayRemainingInsertEventExceedLimit(final String pipeID) {
- if (Objects.isNull(metricService)) {
- return true;
- }
-
- if (remainingEventAndTimeOperatorMap.values().stream()
- .map(PipeDataNodeRemainingEventAndTimeOperator::getRemainingInsertEventSmoothingCount)
- .reduce(0d, Double::sum)
- > PipeConfig.getInstance().getPipeMaxAllowedTotalRemainingInsertEventCount()) {
- return true;
- }
-
- final PipeDataNodeRemainingEventAndTimeOperator operator =
- remainingEventAndTimeOperatorMap.get(pipeID);
- if (Objects.isNull(operator)) {
- LOGGER.warn(
- "Failed to get remaining insert event, RemainingEventAndTimeOperator({}) does not exist, will degrade anyway",
- pipeID);
- return true;
- }
-
- return operator.getRemainingInsertEventSmoothingCount()
- > PipeConfig.getInstance().getPipeMaxAllowedRemainingInsertEventCountPerPipe();
- }
-
@Override
public void unbindFrom(final AbstractMetricService metricService) {
ImmutableSet.copyOf(remainingEventAndTimeOperatorMap.keySet()).forEach(this::deregister);
@@ -184,6 +189,27 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
operator.getPipeName(),
Tag.CREATION_TIME.toString(),
String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_FLOATING_MEMORY_USAGE.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ Tag.NAME.toString(),
+ operator.getPipeName(),
+ Tag.CREATION_TIME.toString(),
+ String.valueOf(operator.getCreationTime()));
metricService.remove(
MetricType.TIMER,
Metric.PIPE_INSERT_NODE_EVENT_TRANSFER_TIME.toString(),
@@ -280,7 +306,7 @@ public class PipeDataNodeSinglePipeMetrics implements IMetricSet {
operator.decreaseTsFileEventCount();
operator.getTsFileTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
- PIPE_DATANODE_EVENT_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+ PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
}
public void increaseHeartbeatEventCount(final String pipeName, final long creationTime) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
index 7d9b172cc92..4156aea4bc5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeResourceMetrics.java
@@ -22,10 +22,9 @@ package org.apache.iotdb.db.pipe.metric.overview;
import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
-import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
@@ -36,6 +35,7 @@ import org.apache.iotdb.metrics.utils.MetricType;
public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_USED_MEMORY = "PipeUsedMemory";
+ private static final String PIPE_USED_FLOATING_MEMORY = "PipeUsedFloatingMemory";
private static final String PIPE_TABLET_USED_MEMORY = "PipeTabletUsedMemory";
@@ -44,6 +44,7 @@ public class PipeResourceMetrics implements IMetricSet {
private static final String PIPE_TOTAL_MEMORY = "PipeTotalMemory";
private Counter diskIOCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ private static final String PIPE_FLOATING_MEMORY = "PipeFloatingMemory";
//////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
@@ -78,22 +79,20 @@ public class PipeResourceMetrics implements IMetricSet {
PipeMemoryManager::getTotalNonFloatingMemorySizeInBytes,
Tag.NAME.toString(),
PIPE_TOTAL_MEMORY);
- // resource reference count
- metricService.createAutoGauge(
- Metric.PIPE_PINNED_MEMTABLE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.wal(),
- PipeWALResourceManager::getPinnedWalCount);
metricService.createAutoGauge(
- Metric.PIPE_LINKED_TSFILE_COUNT.toString(),
+ Metric.PIPE_MEM.toString(),
MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.tsfile(),
- PipeTsFileResourceManager::getLinkedTsfileCount);
+ PipeDataNodeResourceManager.memory(),
+ o -> PipeDataNodeResourceManager.memory().getTotalFloatingMemorySizeInBytes(),
+ Tag.NAME.toString(),
+ PIPE_FLOATING_MEMORY);
metricService.createAutoGauge(
- Metric.PIPE_LINKED_TSFILE_SIZE.toString(),
+ Metric.PIPE_MEM.toString(),
MetricLevel.IMPORTANT,
- PipeDataNodeResourceManager.tsfile(),
- PipeTsFileResourceManager::getTotalLinkedTsfileSize);
+ PipeDataNodeResourceManager.memory(),
+ o -> PipeDataNodeAgent.task().getAllFloatingMemoryUsageInByte(),
+ Tag.NAME.toString(),
+ PIPE_USED_FLOATING_MEMORY);
// phantom reference count
metricService.createAutoGauge(
Metric.PIPE_PHANTOM_REFERENCE_COUNT.toString(),
@@ -123,8 +122,17 @@ public class PipeResourceMetrics implements IMetricSet {
PIPE_TS_FILE_USED_MEMORY);
metricService.remove(
MetricType.AUTO_GAUGE, Metric.PIPE_MEM.toString(), Tag.NAME.toString(), PIPE_TOTAL_MEMORY);
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_MEM.toString(),
+ Tag.NAME.toString(),
+ PIPE_FLOATING_MEMORY);
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ Metric.PIPE_MEM.toString(),
+ Tag.NAME.toString(),
+ PIPE_USED_FLOATING_MEMORY);
// resource reference count
- metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_PINNED_MEMTABLE_COUNT.toString());
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_COUNT.toString());
metricService.remove(MetricType.AUTO_GAUGE, Metric.PIPE_LINKED_TSFILE_SIZE.toString());
// phantom reference count
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
deleted file mode 100644
index b2e605bf84c..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeWALInsertNodeCacheMetrics.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.metric.overview;
-
-import org.apache.iotdb.commons.service.metric.enums.Metric;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
-import org.apache.iotdb.metrics.AbstractMetricService;
-import org.apache.iotdb.metrics.metricsets.IMetricSet;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.MetricType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class PipeWALInsertNodeCacheMetrics implements IMetricSet {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALInsertNodeCacheMetrics.class);
-
- //////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
-
- @Override
- public void bindTo(AbstractMetricService metricService) {
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_RATE.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheHitRate);
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_COUNT.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheHitCount);
- metricService.createAutoGauge(
- Metric.PIPE_WAL_INSERT_NODE_CACHE_REQUEST_COUNT.toString(),
- MetricLevel.IMPORTANT,
- WALInsertNodeCache.getInstance(),
- WALInsertNodeCache::getCacheRequestCount);
- }
-
- @Override
- public void unbindFrom(AbstractMetricService metricService) {
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_RATE.toString());
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_HIT_COUNT.toString());
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PIPE_WAL_INSERT_NODE_CACHE_REQUEST_COUNT.toString());
- }
-
- //////////////////////////// singleton ////////////////////////////
-
- private static class PipeWALInsertNodeCacheMetricsHolder {
-
- private static final PipeWALInsertNodeCacheMetrics INSTANCE =
- new PipeWALInsertNodeCacheMetrics();
-
- private PipeWALInsertNodeCacheMetricsHolder() {
- // empty constructor
- }
- }
-
- public static PipeWALInsertNodeCacheMetrics getInstance() {
- return PipeWALInsertNodeCacheMetricsHolder.INSTANCE;
- }
-
- private PipeWALInsertNodeCacheMetrics() {
- // empty constructor
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java
index 80ebb272c96..49ac24f3e4c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/sink/PipeDataRegionConnectorMetrics.java
@@ -23,7 +23,9 @@ import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtask;
import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingHistogram;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Histogram;
import org.apache.iotdb.metrics.type.Rate;
import org.apache.iotdb.metrics.type.Timer;
import org.apache.iotdb.metrics.utils.MetricLevel;
@@ -44,6 +46,14 @@ public class PipeDataRegionConnectorMetrics implements IMetricSet {
private static final Logger LOGGER =
LoggerFactory.getLogger(PipeDataRegionConnectorMetrics.class);
+ public static Histogram tabletBatchSizeHistogram = new DoNothingHistogram();
+
+ public static Histogram tsFileBatchSizeHistogram = new DoNothingHistogram();
+
+ public static Histogram tabletBatchTimeIntervalHistogram = new DoNothingHistogram();
+
+ public static Histogram tsFileBatchTimeIntervalHistogram = new DoNothingHistogram();
+
@SuppressWarnings("java:S3077")
private volatile AbstractMetricService metricService;
@@ -66,6 +76,22 @@ public class PipeDataRegionConnectorMetrics implements IMetricSet {
for (String taskID : taskIDs) {
createMetrics(taskID);
}
+
+ tabletBatchSizeHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_INSERT_NODE_BATCH_SIZE.toString(), MetricLevel.IMPORTANT);
+
+ tsFileBatchSizeHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_TSFILE_BATCH_SIZE.toString(), MetricLevel.IMPORTANT);
+
+ tabletBatchTimeIntervalHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_INSERT_NODE_BATCH_TIME_COST.toString(), MetricLevel.IMPORTANT);
+
+ tsFileBatchTimeIntervalHistogram =
+ metricService.getOrCreateHistogram(
+ Metric.PIPE_TSFILE_BATCH_TIME_COST.toString(), MetricLevel.IMPORTANT);
}
private void createMetrics(final String taskID) {
@@ -230,6 +256,14 @@ public class PipeDataRegionConnectorMetrics implements IMetricSet {
LOGGER.warn(
"Failed to unbind from pipe data region connector metrics, connector map not empty");
}
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_INSERT_NODE_BATCH_SIZE.toString());
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_TSFILE_BATCH_SIZE.toString());
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_INSERT_NODE_BATCH_TIME_COST.toString());
+
+ metricService.remove(MetricType.HISTOGRAM, Metric.PIPE_TSFILE_BATCH_TIME_COST.toString());
}
private void removeMetrics(final String taskID) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
index cb1ba0b9ad9..0e1f0f2b033 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/CombineRequest.java
@@ -114,7 +114,6 @@ public class CombineRequest extends TPipeTransferReq {
version = transferReq.version;
type = transferReq.type;
- body = transferReq.body;
return this;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
index b20904a0e2b..752be403008 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/exchange/payload/FetchCombineResultRequest.java
@@ -99,7 +99,6 @@ public class FetchCombineResultRequest extends TPipeTransferReq {
version = transferReq.version;
type = transferReq.type;
- body = transferReq.body;
return this;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index 720ecfa76ac..692a029c96d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -208,6 +208,14 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
case HANDSHAKE_DATANODE_V1:
{
try {
+ if (PipeConfig.getInstance().isPipeEnableMemoryCheck()
+ && PipeDataNodeResourceManager.memory().getFreeMemorySizeInBytes()
+ < PipeConfig.getInstance().getPipeMinimumReceiverMemory()) {
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(),
+ "The receiver memory is not enough to handle the handshake request from datanode."));
+ }
return handleTransferHandshakeV1(
PipeTransferDataNodeHandshakeV1Req.fromTPipeTransferReq(req));
} finally {
@@ -218,6 +226,14 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
case HANDSHAKE_DATANODE_V2:
{
try {
+ if (PipeConfig.getInstance().isPipeEnableMemoryCheck()
+ && PipeDataNodeResourceManager.memory().getFreeMemorySizeInBytes()
+ < PipeConfig.getInstance().getPipeMinimumReceiverMemory()) {
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.PIPE_HANDSHAKE_ERROR.getStatusCode(),
+ "The receiver memory is not enough to handle the handshake request from datanode."));
+ }
return handleTransferHandshakeV2(
PipeTransferDataNodeHandshakeV2Req.fromTPipeTransferReq(req));
} finally {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
index 573106e45c2..aaf9eff454b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeResourceManager.java
@@ -19,24 +19,19 @@
package org.apache.iotdb.db.pipe.resource;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.resource.log.PipeLogManager;
import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager;
import org.apache.iotdb.commons.pipe.resource.snapshot.PipeSnapshotResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
import org.apache.iotdb.db.pipe.resource.ref.PipeDataNodePhantomReferenceManager;
import org.apache.iotdb.db.pipe.resource.snapshot.PipeDataNodeSnapshotResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeCompactionManager;
import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.hardlink.PipeWALHardlinkResourceManager;
-import org.apache.iotdb.db.pipe.resource.wal.selfhost.PipeWALSelfHostResourceManager;
-
-import java.util.concurrent.atomic.AtomicReference;
public class PipeDataNodeResourceManager {
private final PipeTsFileResourceManager pipeTsFileResourceManager;
- private final AtomicReference<PipeWALResourceManager> pipeWALResourceManager;
+ private final PipeCompactionManager pipeCompactionManager;
private final PipeSnapshotResourceManager pipeSnapshotResourceManager;
private final PipeMemoryManager pipeMemoryManager;
private final PipeLogManager pipeLogManager;
@@ -46,18 +41,8 @@ public class PipeDataNodeResourceManager {
return PipeResourceManagerHolder.INSTANCE.pipeTsFileResourceManager;
}
- public static PipeWALResourceManager wal() {
- if (PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get() == null) {
- synchronized (PipeResourceManagerHolder.INSTANCE) {
- if (PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get() == null) {
- PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.set(
- PipeConfig.getInstance().getPipeHardLinkWALEnabled()
- ? new PipeWALHardlinkResourceManager()
- : new PipeWALSelfHostResourceManager());
- }
- }
- }
- return PipeResourceManagerHolder.INSTANCE.pipeWALResourceManager.get();
+ public static PipeCompactionManager compaction() {
+ return PipeResourceManagerHolder.INSTANCE.pipeCompactionManager;
}
public static PipeSnapshotResourceManager snapshot() {
@@ -80,7 +65,7 @@ public class PipeDataNodeResourceManager {
private PipeDataNodeResourceManager() {
pipeTsFileResourceManager = new PipeTsFileResourceManager();
- pipeWALResourceManager = new AtomicReference<>();
+ pipeCompactionManager = new PipeCompactionManager();
pipeSnapshotResourceManager = new PipeDataNodeSnapshotResourceManager();
pipeMemoryManager = new PipeMemoryManager();
pipeLogManager = new PipeLogManager();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
index 786a8c63209..aad17113a74 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
@@ -59,9 +59,6 @@ public class PipeMemoryManager {
private volatile long usedMemorySizeInBytesOfTsFiles;
- private static final double FLOATING_MEMORY_RATIO =
- PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion();
-
// Only non-zero memory blocks will be added to this set.
private final Set<PipeMemoryBlock> allocatedBlocks = new HashSet<>();
@@ -101,18 +98,6 @@ public class PipeMemoryManager {
* getTotalNonFloatingMemorySizeInBytes();
}
- public long getAllocatedMemorySizeInBytesOfWAL() {
- return (long)
- (PIPE_CONFIG.getPipeDataStructureWalMemoryProportion()
- * getTotalNonFloatingMemorySizeInBytes());
- }
-
- public long getAllocatedMemorySizeInBytesOfBatch() {
- return (long)
- (PIPE_CONFIG.getPipeDataStructureBatchMemoryProportion()
- * getTotalNonFloatingMemorySizeInBytes());
- }
-
public boolean isEnough4TabletParsing() {
return (double) usedMemorySizeInBytesOfTablets + (double) usedMemorySizeInBytesOfTsFiles
< EXCEED_PROTECT_THRESHOLD * allowedMaxMemorySizeInBytesOfTabletsAndTsFiles()
@@ -646,15 +631,29 @@ public class PipeMemoryManager {
return usedMemorySizeInBytesOfTsFiles;
}
+ public long getAllocatedMemorySizeInBytesOfBatch() {
+ return (long)
+ (PipeConfig.getInstance().getPipeDataStructureBatchMemoryProportion()
+ * getTotalNonFloatingMemorySizeInBytes());
+ }
+
public long getFreeMemorySizeInBytes() {
return memoryBlock.getFreeMemoryInBytes();
}
public long getTotalNonFloatingMemorySizeInBytes() {
- return (long) (memoryBlock.getTotalMemorySizeInBytes() * (1 - FLOATING_MEMORY_RATIO));
+ return (long)
+ (memoryBlock.getTotalMemorySizeInBytes()
+ * (1 - PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion()));
}
public long getTotalFloatingMemorySizeInBytes() {
- return (long) (memoryBlock.getTotalMemorySizeInBytes() * FLOATING_MEMORY_RATIO);
+ return (long)
+ (memoryBlock.getTotalMemorySizeInBytes()
+ * PipeConfig.getInstance().getPipeTotalFloatingMemoryProportion());
+ }
+
+ public long getTotalMemorySizeInBytes() {
+ return memoryBlock.getTotalMemorySizeInBytes();
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
index 2e2c29268ec..30b68e41741 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java
@@ -307,16 +307,16 @@ public class PipeMemoryWeightUtil {
continue;
}
// consider variable references (plus 8) and memory alignment (round up to 8)
- totalSizeInBytes += roundUpToMultiple(primitiveType.getSize() + 8L, 8);
+ totalSizeInBytes += roundUpToMultiple(primitiveType.getSize() + 8, 8);
}
} else {
if (type.isBinary()) {
final Binary binary = batchData.getBinary();
// refer to org.apache.tsfile.utils.TsPrimitiveType.TsBinary.getSize
totalSizeInBytes +=
- roundUpToMultiple((binary == null ? 8 : binary.ramBytesUsed()) + 8L, 8);
+ roundUpToMultiple((binary == null ? 8 : binary.getLength() + 8) + 8, 8);
} else {
- totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8L, 8);
+ totalSizeInBytes += roundUpToMultiple(TsPrimitiveType.getByType(type).getSize() + 8, 8);
}
}
}
@@ -358,7 +358,7 @@ public class PipeMemoryWeightUtil {
* @param n The specified multiple.
* @return The nearest multiple of n greater than or equal to num.
*/
- private static long roundUpToMultiple(long num, int n) {
+ private static int roundUpToMultiple(int num, int n) {
if (n == 0) {
throw new IllegalArgumentException("The multiple n must be greater than 0");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
new file mode 100644
index 00000000000..03860fda8f8
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeCompactionManager.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.resource.tsfile;
+
+import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
+import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtaskLifeCycle;
+import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeRealtimePriorityBlockingQueue;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.pipe.api.event.Event;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+public class PipeCompactionManager {
+
+ private final Set<PipeConnectorSubtaskLifeCycle> pipeConnectorSubtaskLifeCycles =
+ new CopyOnWriteArraySet<>();
+
+ public void registerPipeConnectorSubtaskLifeCycle(
+ final PipeConnectorSubtaskLifeCycle pipeConnectorSubtaskLifeCycle) {
+ pipeConnectorSubtaskLifeCycles.add(pipeConnectorSubtaskLifeCycle);
+ }
+
+ public void deregisterPipeConnectorSubtaskLifeCycle(
+ final PipeConnectorSubtaskLifeCycle pipeConnectorSubtaskLifeCycle) {
+ pipeConnectorSubtaskLifeCycles.remove(pipeConnectorSubtaskLifeCycle);
+ }
+
+ public void emitResult(
+ final String storageGroupName,
+ final String dataRegionId,
+ final long timePartition,
+ final List<TsFileResource> seqFileResources,
+ final List<TsFileResource> unseqFileResources,
+ final List<TsFileResource> targetFileResources) {
+ final Set<TsFileResource> sourceFileResources = new HashSet<>(seqFileResources);
+ sourceFileResources.addAll(unseqFileResources);
+
+ for (final PipeConnectorSubtaskLifeCycle lifeCycle : pipeConnectorSubtaskLifeCycles) {
+ final UnboundedBlockingPendingQueue<Event> pendingQueue = lifeCycle.getPendingQueue();
+ if (pendingQueue instanceof PipeRealtimePriorityBlockingQueue) {
+ final PipeRealtimePriorityBlockingQueue realtimePriorityBlockingQueue =
+ (PipeRealtimePriorityBlockingQueue) pendingQueue;
+ realtimePriorityBlockingQueue.replace(
+ dataRegionId, sourceFileResources, targetFileResources);
+ }
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFilePublicResource.java
similarity index 63%
copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFilePublicResource.java
index bf789b9732d..47134fe117c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFilePublicResource.java
@@ -23,8 +23,6 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -36,106 +34,27 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-public class PipeTsFileResource implements AutoCloseable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResource.class);
-
- public static final long TSFILE_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
+public class PipeTsFilePublicResource extends PipeTsFileResource {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFilePublicResource.class);
public static final float MEMORY_SUFFICIENT_THRESHOLD = 0.7f;
-
- private final File hardlinkOrCopiedFile;
- private final boolean isTsFile;
-
- /** this TsFileResource is used to track the {@link TsFileResourceStatus} of original TsFile. * */
- private final TsFileResource tsFileResource;
-
- private volatile long fileSize = -1L;
-
- private final AtomicInteger referenceCount;
- private final AtomicLong lastUnpinToZeroTime;
private PipeMemoryBlock allocatedMemoryBlock;
private Map<IDeviceID, List<String>> deviceMeasurementsMap = null;
private Map<IDeviceID, Boolean> deviceIsAlignedMap = null;
private Map<String, TSDataType> measurementDataTypeMap = null;
- public PipeTsFileResource(
- final File hardlinkOrCopiedFile,
- final boolean isTsFile,
- final TsFileResource tsFileResource) {
- this.hardlinkOrCopiedFile = hardlinkOrCopiedFile;
- this.isTsFile = isTsFile;
- this.tsFileResource = tsFileResource;
-
- referenceCount = new AtomicInteger(1);
- lastUnpinToZeroTime = new AtomicLong(Long.MAX_VALUE);
- }
-
- public File getFile() {
- return hardlinkOrCopiedFile;
- }
-
- public boolean isOriginalTsFileDeleted() {
- return isTsFile && Objects.nonNull(tsFileResource) && tsFileResource.isDeleted();
- }
-
- public long getFileSize() {
- if (fileSize == -1L) {
- synchronized (this) {
- if (fileSize == -1L) {
- fileSize = hardlinkOrCopiedFile.length();
- }
- }
- }
- return fileSize;
- }
-
- public long getTsFileResourceSize() {
- return Objects.nonNull(tsFileResource) ? tsFileResource.calculateRamSize() : 0;
- }
-
- ///////////////////// Reference Count /////////////////////
-
- public int getReferenceCount() {
- return referenceCount.get();
- }
-
- public int increaseAndGetReference() {
- return referenceCount.addAndGet(1);
- }
-
- public int decreaseAndGetReference() {
- final int finalReferenceCount = referenceCount.addAndGet(-1);
- if (finalReferenceCount == 0) {
- lastUnpinToZeroTime.set(System.currentTimeMillis());
- }
- if (finalReferenceCount < 0) {
- LOGGER.warn("PipeTsFileResource's reference count is decreased to below 0.");
- }
- return finalReferenceCount;
- }
-
- public synchronized boolean closeIfOutOfTimeToLive() {
- if (referenceCount.get() <= 0
- && (deviceMeasurementsMap == null // Not cached yet.
- || System.currentTimeMillis() - lastUnpinToZeroTime.get()
- > TSFILE_MIN_TIME_TO_LIVE_IN_MS)) {
- close();
- return true;
- } else {
- return false;
- }
+ public PipeTsFilePublicResource(File hardlinkOrCopiedFile) {
+ super(hardlinkOrCopiedFile);
}
@Override
- public synchronized void close() {
+ public void close() {
+ super.close();
+
if (deviceMeasurementsMap != null) {
deviceMeasurementsMap = null;
}
@@ -152,53 +71,39 @@ public class PipeTsFileResource implements AutoCloseable {
allocatedMemoryBlock.close();
allocatedMemoryBlock = null;
}
-
- try {
- Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
- } catch (final Exception e) {
- LOGGER.error(
- "PipeTsFileResource: Failed to delete tsfile {} when closing, because {}. Please MANUALLY delete it.",
- hardlinkOrCopiedFile,
- e.getMessage(),
- e);
- }
-
- LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
}
//////////////////////////// Cache Getter ////////////////////////////
- public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap()
+ public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap(final File tsFile)
throws IOException {
- if (deviceMeasurementsMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ if (deviceMeasurementsMap == null) {
+ cacheObjectsIfAbsent(tsFile);
}
return deviceMeasurementsMap;
}
public synchronized Map<IDeviceID, Boolean> tryGetDeviceIsAlignedMap(
- final boolean cacheOtherMetadata) throws IOException {
- if (deviceIsAlignedMap == null && isTsFile) {
+ final boolean cacheOtherMetadata, final File tsFile) throws IOException {
+ if (deviceIsAlignedMap == null) {
if (cacheOtherMetadata) {
- cacheObjectsIfAbsent();
+ cacheObjectsIfAbsent(tsFile);
} else {
- cacheDeviceIsAlignedMapIfAbsent();
+ cacheDeviceIsAlignedMapIfAbsent(tsFile);
}
}
return deviceIsAlignedMap;
}
- public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap() throws IOException {
- if (measurementDataTypeMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap(final File tsFile)
+ throws IOException {
+ if (measurementDataTypeMap == null) {
+ cacheObjectsIfAbsent(tsFile);
}
return measurementDataTypeMap;
}
- synchronized boolean cacheDeviceIsAlignedMapIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
+ synchronized boolean cacheDeviceIsAlignedMapIfAbsent(final File tsFile) throws IOException {
if (allocatedMemoryBlock != null) {
// This means objects are already cached.
@@ -216,13 +121,13 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"Failed to cacheDeviceIsAlignedMapIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
return false;
}
long memoryRequiredInBytes = 0L;
try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, false)) {
+ new TsFileSequenceReader(tsFile.getPath(), true, false)) {
deviceIsAlignedMap = new HashMap<>();
final TsFileDeviceIterator deviceIsAlignedIterator =
sequenceReader.getAllDevicesIteratorWithIsAligned();
@@ -243,22 +148,16 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
deviceIsAlignedMap = null;
return false;
}
- LOGGER.info(
- "PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.",
- hardlinkOrCopiedFile.getPath());
+ LOGGER.info("PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.", tsFile.getPath());
return true;
}
- synchronized boolean cacheObjectsIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
+ synchronized boolean cacheObjectsIfAbsent(final File tsFile) throws IOException {
if (allocatedMemoryBlock != null) {
if (deviceMeasurementsMap != null) {
return true;
@@ -280,13 +179,13 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"Failed to cacheObjectsIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
return false;
}
long memoryRequiredInBytes = 0L;
try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, true)) {
+ new TsFileSequenceReader(tsFile.getPath(), true, true)) {
deviceMeasurementsMap = sequenceReader.getDeviceMeasurementsMap();
memoryRequiredInBytes +=
PipeMemoryWeightUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap);
@@ -316,15 +215,14 @@ public class PipeTsFileResource implements AutoCloseable {
if (allocatedMemoryBlock == null) {
LOGGER.info(
"PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
+ tsFile.getPath());
deviceIsAlignedMap = null;
deviceMeasurementsMap = null;
measurementDataTypeMap = null;
return false;
}
- LOGGER.info(
- "PipeTsFileResource: Cached objects for tsfile {}.", hardlinkOrCopiedFile.getPath());
+ LOGGER.info("PipeTsFileResource: Cached objects for tsfile {}.", tsFile.getPath());
return true;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
index bf789b9732d..8b37f877094 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResource.java
@@ -19,73 +19,33 @@
package org.apache.iotdb.db.pipe.resource.tsfile;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
-
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.read.TsFileDeviceIterator;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
import java.nio.file.Files;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
public class PipeTsFileResource implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResource.class);
- public static final long TSFILE_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
- public static final float MEMORY_SUFFICIENT_THRESHOLD = 0.7f;
-
private final File hardlinkOrCopiedFile;
- private final boolean isTsFile;
-
- /** this TsFileResource is used to track the {@link TsFileResourceStatus} of original TsFile. * */
- private final TsFileResource tsFileResource;
private volatile long fileSize = -1L;
private final AtomicInteger referenceCount;
- private final AtomicLong lastUnpinToZeroTime;
- private PipeMemoryBlock allocatedMemoryBlock;
- private Map<IDeviceID, List<String>> deviceMeasurementsMap = null;
- private Map<IDeviceID, Boolean> deviceIsAlignedMap = null;
- private Map<String, TSDataType> measurementDataTypeMap = null;
- public PipeTsFileResource(
- final File hardlinkOrCopiedFile,
- final boolean isTsFile,
- final TsFileResource tsFileResource) {
+ public PipeTsFileResource(final File hardlinkOrCopiedFile) {
this.hardlinkOrCopiedFile = hardlinkOrCopiedFile;
- this.isTsFile = isTsFile;
- this.tsFileResource = tsFileResource;
referenceCount = new AtomicInteger(1);
- lastUnpinToZeroTime = new AtomicLong(Long.MAX_VALUE);
}
public File getFile() {
return hardlinkOrCopiedFile;
}
- public boolean isOriginalTsFileDeleted() {
- return isTsFile && Objects.nonNull(tsFileResource) && tsFileResource.isDeleted();
- }
-
public long getFileSize() {
if (fileSize == -1L) {
synchronized (this) {
@@ -97,64 +57,33 @@ public class PipeTsFileResource implements AutoCloseable {
return fileSize;
}
- public long getTsFileResourceSize() {
- return Objects.nonNull(tsFileResource) ? tsFileResource.calculateRamSize() : 0;
- }
-
///////////////////// Reference Count /////////////////////
public int getReferenceCount() {
return referenceCount.get();
}
- public int increaseAndGetReference() {
- return referenceCount.addAndGet(1);
+ public void increaseReferenceCount() {
+ referenceCount.addAndGet(1);
}
- public int decreaseAndGetReference() {
+ public boolean decreaseReferenceCount() {
final int finalReferenceCount = referenceCount.addAndGet(-1);
if (finalReferenceCount == 0) {
- lastUnpinToZeroTime.set(System.currentTimeMillis());
+ close();
+ return true;
}
if (finalReferenceCount < 0) {
LOGGER.warn("PipeTsFileResource's reference count is decreased to below 0.");
}
- return finalReferenceCount;
- }
-
- public synchronized boolean closeIfOutOfTimeToLive() {
- if (referenceCount.get() <= 0
- && (deviceMeasurementsMap == null // Not cached yet.
- || System.currentTimeMillis() - lastUnpinToZeroTime.get()
- > TSFILE_MIN_TIME_TO_LIVE_IN_MS)) {
- close();
- return true;
- } else {
- return false;
- }
+ return false;
}
@Override
public synchronized void close() {
- if (deviceMeasurementsMap != null) {
- deviceMeasurementsMap = null;
- }
-
- if (deviceIsAlignedMap != null) {
- deviceIsAlignedMap = null;
- }
-
- if (measurementDataTypeMap != null) {
- measurementDataTypeMap = null;
- }
-
- if (allocatedMemoryBlock != null) {
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
- }
-
+ boolean successful = false;
try {
- Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
+ successful = Files.deleteIfExists(hardlinkOrCopiedFile.toPath());
} catch (final Exception e) {
LOGGER.error(
"PipeTsFileResource: Failed to delete tsfile {} when closing, because {}. Please MANUALLY delete it.",
@@ -163,168 +92,8 @@ public class PipeTsFileResource implements AutoCloseable {
e);
}
- LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
- }
-
- //////////////////////////// Cache Getter ////////////////////////////
-
- public synchronized Map<IDeviceID, List<String>> tryGetDeviceMeasurementsMap()
- throws IOException {
- if (deviceMeasurementsMap == null && isTsFile) {
- cacheObjectsIfAbsent();
- }
- return deviceMeasurementsMap;
- }
-
- public synchronized Map<IDeviceID, Boolean> tryGetDeviceIsAlignedMap(
- final boolean cacheOtherMetadata) throws IOException {
- if (deviceIsAlignedMap == null && isTsFile) {
- if (cacheOtherMetadata) {
- cacheObjectsIfAbsent();
- } else {
- cacheDeviceIsAlignedMapIfAbsent();
- }
- }
- return deviceIsAlignedMap;
- }
-
- public synchronized Map<String, TSDataType> tryGetMeasurementDataTypeMap() throws IOException {
- if (measurementDataTypeMap == null && isTsFile) {
- cacheObjectsIfAbsent();
+ if (successful) {
+ LOGGER.info("PipeTsFileResource: Closed tsfile {} and cleaned up.", hardlinkOrCopiedFile);
}
- return measurementDataTypeMap;
- }
-
- synchronized boolean cacheDeviceIsAlignedMapIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
- if (allocatedMemoryBlock != null) {
- // This means objects are already cached.
- return true;
- }
-
- // See if pipe memory is sufficient to be allocated for TsFileSequenceReader.
- // Only allocate when pipe memory used is less than 50%, because memory here
- // is hard to shrink and may consume too much memory.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(
- PipeConfig.getInstance().getPipeMemoryAllocateForTsFileSequenceReaderInBytes(),
- MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "Failed to cacheDeviceIsAlignedMapIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- return false;
- }
-
- long memoryRequiredInBytes = 0L;
- try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, false)) {
- deviceIsAlignedMap = new HashMap<>();
- final TsFileDeviceIterator deviceIsAlignedIterator =
- sequenceReader.getAllDevicesIteratorWithIsAligned();
- while (deviceIsAlignedIterator.hasNext()) {
- final Pair<IDeviceID, Boolean> deviceIsAlignedPair = deviceIsAlignedIterator.next();
- deviceIsAlignedMap.put(deviceIsAlignedPair.getLeft(), deviceIsAlignedPair.getRight());
- }
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfIDeviceId2Bool(deviceIsAlignedMap);
- }
- // Release memory of TsFileSequenceReader.
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
-
- // Allocate again for the cached objects.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(memoryRequiredInBytes, MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- deviceIsAlignedMap = null;
- return false;
- }
-
- LOGGER.info(
- "PipeTsFileResource: Cached deviceIsAlignedMap for tsfile {}.",
- hardlinkOrCopiedFile.getPath());
- return true;
- }
-
- synchronized boolean cacheObjectsIfAbsent() throws IOException {
- if (!isTsFile) {
- return false;
- }
-
- if (allocatedMemoryBlock != null) {
- if (deviceMeasurementsMap != null) {
- return true;
- } else {
- // Recalculate it again because only deviceIsAligned map is cached
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
- }
- }
-
- // See if pipe memory is sufficient to be allocated for TsFileSequenceReader.
- // Only allocate when pipe memory used is less than 50%, because memory here
- // is hard to shrink and may consume too much memory.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(
- PipeConfig.getInstance().getPipeMemoryAllocateForTsFileSequenceReaderInBytes(),
- MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "Failed to cacheObjectsIfAbsent for tsfile {}, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- return false;
- }
-
- long memoryRequiredInBytes = 0L;
- try (TsFileSequenceReader sequenceReader =
- new TsFileSequenceReader(hardlinkOrCopiedFile.getPath(), true, true)) {
- deviceMeasurementsMap = sequenceReader.getDeviceMeasurementsMap();
- memoryRequiredInBytes +=
- PipeMemoryWeightUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap);
-
- if (Objects.isNull(deviceIsAlignedMap)) {
- deviceIsAlignedMap = new HashMap<>();
- final TsFileDeviceIterator deviceIsAlignedIterator =
- sequenceReader.getAllDevicesIteratorWithIsAligned();
- while (deviceIsAlignedIterator.hasNext()) {
- final Pair<IDeviceID, Boolean> deviceIsAlignedPair = deviceIsAlignedIterator.next();
- deviceIsAlignedMap.put(deviceIsAlignedPair.getLeft(), deviceIsAlignedPair.getRight());
- }
- }
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfIDeviceId2Bool(deviceIsAlignedMap);
-
- measurementDataTypeMap = sequenceReader.getFullPathDataTypeMap();
- memoryRequiredInBytes += PipeMemoryWeightUtil.memoryOfStr2TSDataType(measurementDataTypeMap);
- }
- // Release memory of TsFileSequenceReader.
- allocatedMemoryBlock.close();
- allocatedMemoryBlock = null;
-
- // Allocate again for the cached objects.
- allocatedMemoryBlock =
- PipeDataNodeResourceManager.memory()
- .forceAllocateIfSufficient(memoryRequiredInBytes, MEMORY_SUFFICIENT_THRESHOLD);
- if (allocatedMemoryBlock == null) {
- LOGGER.info(
- "PipeTsFileResource: Failed to cache objects for tsfile {} in cache, because memory usage is high",
- hardlinkOrCopiedFile.getPath());
- deviceIsAlignedMap = null;
- deviceMeasurementsMap = null;
- measurementDataTypeMap = null;
- return false;
- }
-
- LOGGER.info(
- "PipeTsFileResource: Cached objects for tsfile {}.", hardlinkOrCopiedFile.getPath());
- return true;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
index 6d04e909cb1..31bc403b5b9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
@@ -22,8 +22,7 @@ package org.apache.iotdb.db.pipe.resource.tsfile;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.enums.TSDataType;
@@ -31,88 +30,30 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
import java.io.File;
import java.io.IOException;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
+import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
public class PipeTsFileResourceManager {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileResourceManager.class);
- private final Map<String, PipeTsFileResource> hardlinkOrCopiedFileToPipeTsFileResourceMap =
- new ConcurrentHashMap<>();
- private final PipeTsFileResourceSegmentLock segmentLock = new PipeTsFileResourceSegmentLock();
-
- public PipeTsFileResourceManager() {
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "PipeTsFileResourceManager#ttlCheck()",
- this::tryTtlCheck,
- Math.max(PipeTsFileResource.TSFILE_MIN_TIME_TO_LIVE_IN_MS / 1000, 1));
- }
-
- private void tryTtlCheck() {
- try {
- ttlCheck();
- } catch (final InterruptedException e) {
- Thread.currentThread().interrupt();
- LOGGER.warn("failed to try lock when checking TTL because of interruption", e);
- } catch (final Exception e) {
- LOGGER.warn("failed to check TTL of PipeTsFileResource: ", e);
- }
- }
-
- private void ttlCheck() throws InterruptedException {
- final Iterator<Map.Entry<String, PipeTsFileResource>> iterator =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.entrySet().iterator();
- final long timeout =
- PipeConfig.getInstance().getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds() >> 1;
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeTsFileResourceManager.class,
- PipeConfig.getInstance().getPipeTsFilePinMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeTsFilePinMaxLogIntervalRounds(),
- hardlinkOrCopiedFileToPipeTsFileResourceMap.size());
- final StringBuilder logBuilder = new StringBuilder();
- while (iterator.hasNext()) {
- final Map.Entry<String, PipeTsFileResource> entry = iterator.next();
-
- final String hardlinkOrCopiedFile = entry.getKey();
- if (!segmentLock.tryLock(new File(hardlinkOrCopiedFile), timeout, TimeUnit.SECONDS)) {
- LOGGER.warn(
- "failed to try lock when checking TTL for file {} because of timeout ({}s)",
- hardlinkOrCopiedFile,
- timeout);
- continue;
- }
+ // This is used to hold the assigner pinned tsFiles.
+ // Also, it is used to provide metadata cache of the tsFile, and is shared by all the pipe's
+ // tsFiles.
+ private final Map<String, PipeTsFilePublicResource>
+ hardlinkOrCopiedFileToTsFilePublicResourceMap = new ConcurrentHashMap<>();
- try {
- if (entry.getValue().closeIfOutOfTimeToLive()) {
- iterator.remove();
- } else {
- logBuilder.append(
- String.format(
- "<%s , %d times, %d bytes> ",
- entry.getKey(),
- entry.getValue().getReferenceCount(),
- entry.getValue().getFileSize()));
- }
- } catch (final Exception e) {
- LOGGER.warn("failed to close PipeTsFileResource when checking TTL: ", e);
- } finally {
- segmentLock.unlock(new File(hardlinkOrCopiedFile));
- }
- }
- if (logBuilder.length() > 0) {
- logger.ifPresent(l -> l.info("Pipe file {}are still referenced", logBuilder));
- }
- }
+ // PipeName -> TsFilePath -> PipeTsFileResource
+ private final Map<String, Map<String, PipeTsFileResource>>
+ hardlinkOrCopiedFileToPipeTsFileResourceMap = new ConcurrentHashMap<>();
+ private final PipeTsFileResourceSegmentLock segmentLock = new PipeTsFileResourceSegmentLock();
/**
* Given a file, create a hardlink or copy it to pipe dir, maintain a reference count for the
@@ -130,19 +71,16 @@ public class PipeTsFileResourceManager {
* @param file tsfile, resource file or mod file. can be original file or hardlink/copy of
* original file
* @param isTsFile {@code true} to create hardlink, {@code false} to copy file
- * @param tsFileResource the TsFileResource of original TsFile. Ignored if {@param isTsFile} is
- * {@code false}.
* @return the hardlink or copied file
* @throws IOException when create hardlink or copy file failed
*/
public File increaseFileReference(
- final File file, final boolean isTsFile, final TsFileResource tsFileResource)
- throws IOException {
+ final File file, final boolean isTsFile, final @Nullable String pipeName) throws IOException {
// If the file is already a hardlink or copied file,
// just increase reference count and return it
segmentLock.lock(file);
try {
- if (increaseReferenceIfExists(file)) {
+ if (increaseReferenceIfExists(file, pipeName)) {
return file;
}
} finally {
@@ -151,13 +89,11 @@ public class PipeTsFileResourceManager {
// If the file is not a hardlink or copied file, check if there is a related hardlink or
// copied file in pipe dir. if so, increase reference count and return it
- final File hardlinkOrCopiedFile = getHardlinkOrCopiedFileInPipeDir(file);
+ final File hardlinkOrCopiedFile = getHardlinkOrCopiedFileInPipeDir(file, pipeName);
segmentLock.lock(hardlinkOrCopiedFile);
try {
- if (increaseReferenceIfExists(hardlinkOrCopiedFile)) {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap
- .get(hardlinkOrCopiedFile.getPath())
- .getFile();
+ if (increaseReferenceIfExists(hardlinkOrCopiedFile, pipeName)) {
+ return getResourceMap(pipeName).get(hardlinkOrCopiedFile.getPath()).getFile();
}
// If the file is a tsfile, create a hardlink in pipe dir and will return it.
@@ -170,27 +106,57 @@ public class PipeTsFileResourceManager {
// If the file is not a hardlink or copied file, and there is no related hardlink or copied
// file in pipe dir, create a hardlink or copy it to pipe dir, maintain a reference count for
// the hardlink or copied file, and return the hardlink or copied file.
- hardlinkOrCopiedFileToPipeTsFileResourceMap.put(
- resultFile.getPath(), new PipeTsFileResource(resultFile, isTsFile, tsFileResource));
+ if (Objects.nonNull(pipeName)) {
+ hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
+ .put(resultFile.getPath(), new PipeTsFileResource(resultFile));
+ } else {
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.put(
+ resultFile.getPath(), new PipeTsFilePublicResource(resultFile));
+ }
+
+ increasePublicReference(resultFile, pipeName);
+
return resultFile;
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
}
}
- private boolean increaseReferenceIfExists(final File file) {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(file.getPath());
+ private boolean increaseReferenceIfExists(final File file, final @Nullable String pipeName) {
+ final String path = file.getPath();
+ final PipeTsFileResource resource = getResourceMap(pipeName).get(path);
if (resource != null) {
- resource.increaseAndGetReference();
+ resource.increaseReferenceCount();
+ increasePublicReference(file, pipeName);
return true;
}
return false;
}
- public static File getHardlinkOrCopiedFileInPipeDir(final File file) throws IOException {
+ private void increasePublicReference(final File file, final String pipeName) {
+ if (Objects.isNull(pipeName)) {
+ return;
+ }
+ // Increase the assigner's file to avoid hard-link or memory cache cleaning
+ // Note that it does not exist for historical files
+ final String path = getCommonFilePath(file);
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.compute(
+ path,
+ (k, v) -> {
+ if (Objects.isNull(v)) {
+ return new PipeTsFilePublicResource(new File(path));
+ } else {
+ v.increaseReferenceCount();
+ return v;
+ }
+ });
+ }
+
+ public static File getHardlinkOrCopiedFileInPipeDir(
+ final File file, final @Nullable String pipeName) throws IOException {
try {
- return new File(getPipeTsFileDirPath(file), getRelativeFilePath(file));
+ return new File(getPipeTsFileDirPath(file, pipeName), getRelativeFilePath(file));
} catch (final Exception e) {
throw new IOException(
String.format(
@@ -201,22 +167,28 @@ public class PipeTsFileResourceManager {
}
}
- private static String getPipeTsFileDirPath(File file) throws IOException {
+ private static String getPipeTsFileDirPath(File file, final @Nullable String pipeName)
+ throws IOException {
while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
- && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)) {
+ && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
+ && !file.getName().equals(PipeConfig.getInstance().getPipeHardlinkBaseDirName())) {
file = file.getParentFile();
}
return file.getParentFile().getCanonicalPath()
+ File.separator
+ PipeConfig.getInstance().getPipeHardlinkBaseDirName()
+ File.separator
- + PipeConfig.getInstance().getPipeHardlinkTsFileDirName();
+ + PipeConfig.getInstance().getPipeHardlinkTsFileDirName()
+ + (Objects.nonNull(pipeName) ? File.separator + pipeName : "");
}
private static String getRelativeFilePath(File file) {
StringBuilder builder = new StringBuilder(file.getName());
while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
- && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)) {
+ && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
+ && !file.getParentFile()
+ .getName()
+ .equals(PipeConfig.getInstance().getPipeHardlinkTsFileDirName())) {
file = file.getParentFile();
builder =
new StringBuilder(file.getName())
@@ -230,32 +202,68 @@ public class PipeTsFileResourceManager {
* Given a hardlink or copied file, decrease its reference count, if the reference count is 0,
* delete the file. if the given file is not a hardlink or copied file, do nothing.
*
- * @param hardlinkOrCopiedFile the copied or hardlinked file
+ * @param hardlinkOrCopiedFile the copied or hard-linked file
*/
- public void decreaseFileReference(final File hardlinkOrCopiedFile) {
+ public void decreaseFileReference(
+ final File hardlinkOrCopiedFile, final @Nullable String pipeName) {
segmentLock.lock(hardlinkOrCopiedFile);
try {
final String filePath = hardlinkOrCopiedFile.getPath();
- final PipeTsFileResource resource = hardlinkOrCopiedFileToPipeTsFileResourceMap.get(filePath);
- if (resource != null) {
- resource.decreaseAndGetReference();
+ final PipeTsFileResource resource = getResourceMap(pipeName).get(filePath);
+ if (resource != null && resource.decreaseReferenceCount()) {
+ getResourceMap(pipeName).remove(filePath);
}
+ // Decrease the assigner's file to clear hard-link and memory cache
+ // Note that it does not exist for historical files
+ decreasePublicReferenceIfExists(hardlinkOrCopiedFile, pipeName);
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
}
}
+ private void decreasePublicReferenceIfExists(final File file, final @Nullable String pipeName) {
+ if (Objects.isNull(pipeName)) {
+ return;
+ }
+ // Increase the assigner's file to avoid hard-link or memory cache cleaning
+ // Note that it does not exist for historical files
+ final String commonFilePath = getCommonFilePath(file);
+ if (hardlinkOrCopiedFileToTsFilePublicResourceMap.containsKey(commonFilePath)
+ && hardlinkOrCopiedFileToTsFilePublicResourceMap
+ .get(commonFilePath)
+ .decreaseReferenceCount()) {
+ hardlinkOrCopiedFileToPipeTsFileResourceMap.remove(commonFilePath);
+ }
+ }
+
+ // Warning: Shall not be called by the assigner
+ private String getCommonFilePath(final @Nonnull File file) {
+ // If the parent or grandparent is null then this is testing scenario
+ // Skip the "pipeName" of this file
+ return Objects.isNull(file.getParentFile())
+ || Objects.isNull(file.getParentFile().getParentFile())
+ ? file.getPath()
+ : file.getParentFile().getParent() + File.separator + file.getName();
+ }
+
/**
* Get the reference count of the file.
*
* @param hardlinkOrCopiedFile the copied or hardlinked file
* @return the reference count of the file
*/
- public int getFileReferenceCount(final File hardlinkOrCopiedFile) {
+ @TestOnly
+ public int getFileReferenceCount(
+ final File hardlinkOrCopiedFile, final @Nullable String pipeName) {
segmentLock.lock(hardlinkOrCopiedFile);
try {
- final String filePath = hardlinkOrCopiedFile.getPath();
- final PipeTsFileResource resource = hardlinkOrCopiedFileToPipeTsFileResourceMap.get(filePath);
+ final PipeTsFileResource resource =
+ Objects.nonNull(pipeName)
+ ? hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .get(hardlinkOrCopiedFile.getPath())
+ : hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedFile));
return resource != null ? resource.getReferenceCount() : 0;
} finally {
segmentLock.unlock(hardlinkOrCopiedFile);
@@ -271,9 +279,14 @@ public class PipeTsFileResourceManager {
public boolean cacheObjectsIfAbsent(final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource != null && resource.cacheObjectsIfAbsent();
+ if (hardlinkOrCopiedTsFile.getParentFile() == null
+ || hardlinkOrCopiedTsFile.getParentFile().getParentFile() == null) {
+ return false;
+ }
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource != null && resource.cacheObjectsIfAbsent(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -283,9 +296,10 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetDeviceMeasurementsMap();
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null ? null : resource.tryGetDeviceMeasurementsMap(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -295,9 +309,12 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile, final boolean cacheOtherMetadata) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetDeviceIsAlignedMap(cacheOtherMetadata);
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null
+ ? null
+ : resource.tryGetDeviceIsAlignedMap(cacheOtherMetadata, hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
@@ -307,37 +324,54 @@ public class PipeTsFileResourceManager {
final File hardlinkOrCopiedTsFile) throws IOException {
segmentLock.lock(hardlinkOrCopiedTsFile);
try {
- final PipeTsFileResource resource =
- hardlinkOrCopiedFileToPipeTsFileResourceMap.get(hardlinkOrCopiedTsFile.getPath());
- return resource == null ? null : resource.tryGetMeasurementDataTypeMap();
+ final PipeTsFilePublicResource resource =
+ hardlinkOrCopiedFileToTsFilePublicResourceMap.get(
+ getCommonFilePath(hardlinkOrCopiedTsFile));
+ return resource == null
+ ? null
+ : resource.tryGetMeasurementDataTypeMap(hardlinkOrCopiedTsFile);
} finally {
segmentLock.unlock(hardlinkOrCopiedTsFile);
}
}
- public void pinTsFileResource(final TsFileResource resource, final boolean withMods)
+ public Map<String, ? extends PipeTsFileResource> getResourceMap(final @Nullable String pipeName) {
+ return Objects.nonNull(pipeName)
+ ? hardlinkOrCopiedFileToPipeTsFileResourceMap.computeIfAbsent(
+ pipeName, k -> new ConcurrentHashMap<>())
+ : hardlinkOrCopiedFileToTsFilePublicResourceMap;
+ }
+
+ public void pinTsFileResource(
+ final TsFileResource resource, final boolean withMods, final @Nullable String pipeName)
throws IOException {
- increaseFileReference(resource.getTsFile(), true, resource);
+ increaseFileReference(resource.getTsFile(), true, pipeName);
if (withMods && resource.getExclusiveModFile().exists()) {
- increaseFileReference(resource.getExclusiveModFile().getFile(), false, null);
+ increaseFileReference(resource.getExclusiveModFile().getFile(), false, pipeName);
}
}
- public void unpinTsFileResource(final TsFileResource resource) throws IOException {
- final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile());
- decreaseFileReference(pinnedFile);
+ public void unpinTsFileResource(final TsFileResource resource, final @Nullable String pipeName)
+ throws IOException {
+ final File pinnedFile = getHardlinkOrCopiedFileInPipeDir(resource.getTsFile(), pipeName);
+ decreaseFileReference(pinnedFile, pipeName);
if (resource.sharedModFileExists()) {
- decreaseFileReference(resource.getSharedModFile().getFile());
+ decreaseFileReference(resource.getSharedModFile().getFile(), pipeName);
}
}
- public int getLinkedTsfileCount() {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.size();
+ public int getLinkedTsFileCount(final @Nonnull String pipeName) {
+ return hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .size();
}
- public long getTotalLinkedTsfileSize() {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.values().stream()
+ public long getTotalLinkedTsFileSize(final @Nonnull String pipeName) {
+ return hardlinkOrCopiedFileToPipeTsFileResourceMap
+ .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
+ .values()
+ .stream()
.mapToLong(
resource -> {
try {
@@ -349,47 +383,4 @@ public class PipeTsFileResourceManager {
})
.sum();
}
-
- /**
- * Get the total size of linked TsFiles whose original TsFile is deleted (by compaction or else)
- */
- public long getTotalLinkedButDeletedTsfileSize() {
- try {
- return hardlinkOrCopiedFileToPipeTsFileResourceMap.values().parallelStream()
- .filter(PipeTsFileResource::isOriginalTsFileDeleted)
- .mapToLong(
- resource -> {
- try {
- return resource.getFileSize();
- } catch (Exception e) {
- LOGGER.warn(
- "failed to get file size of linked but deleted TsFile {}: ", resource, e);
- return 0;
- }
- })
- .sum();
- } catch (final Exception e) {
- LOGGER.warn("failed to get total size of linked but deleted TsFiles: ", e);
- return 0;
- }
- }
-
- public long getTotalLinkedButDeletedTsFileResourceRamSize() {
- long totalLinkedButDeletedTsfileResourceRamSize = 0;
- try {
- for (final Map.Entry<String, PipeTsFileResource> resourceEntry :
- hardlinkOrCopiedFileToPipeTsFileResourceMap.entrySet()) {
- final PipeTsFileResource pipeTsFileResource = resourceEntry.getValue();
- // If the original TsFile is not deleted, the memory of the resource is not counted
- // because the memory of the resource is controlled by TsFileResourceManager.
- if (pipeTsFileResource.isOriginalTsFileDeleted()) {
- totalLinkedButDeletedTsfileResourceRamSize += pipeTsFileResource.getTsFileResourceSize();
- }
- }
- return totalLinkedButDeletedTsfileResourceRamSize;
- } catch (final Exception e) {
- LOGGER.warn("failed to get total size of linked but deleted TsFiles resource ram size: ", e);
- return totalLinkedButDeletedTsfileResourceRamSize;
- }
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
deleted file mode 100644
index 9d1e530a19d..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResource.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal;
-
-import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException;
-import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-public abstract class PipeWALResource implements Closeable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALResource.class);
-
- protected final WALEntryHandler walEntryHandler;
-
- private final AtomicInteger referenceCount;
-
- public static final long WAL_MIN_TIME_TO_LIVE_IN_MS = 1000L * 20;
- private final AtomicLong lastLogicalPinTime;
- private final AtomicBoolean isPhysicallyPinned;
-
- protected PipeWALResource(WALEntryHandler walEntryHandler) {
- this.walEntryHandler = walEntryHandler;
-
- referenceCount = new AtomicInteger(0);
-
- lastLogicalPinTime = new AtomicLong(0);
- isPhysicallyPinned = new AtomicBoolean(false);
- }
-
- public final void pin() throws PipeRuntimeNonCriticalException {
- if (referenceCount.get() == 0) {
- if (!isPhysicallyPinned.get()) {
- try {
- pinInternal();
- } catch (MemTablePinException e) {
- throw new PipeRuntimeNonCriticalException(
- String.format(
- "failed to pin wal %d, because %s",
- walEntryHandler.getMemTableId(), e.getMessage()));
- }
- isPhysicallyPinned.set(true);
- LOGGER.info("wal {} is pinned by pipe engine", walEntryHandler.getMemTableId());
- } // else means the wal is already pinned, do nothing
-
- // no matter the wal is pinned or not, update the last pin time
- lastLogicalPinTime.set(System.currentTimeMillis());
- }
-
- referenceCount.incrementAndGet();
- }
-
- protected abstract void pinInternal()
- throws MemTablePinException, PipeRuntimeNonCriticalException;
-
- public final void unpin() throws PipeRuntimeNonCriticalException {
- final int finalReferenceCount = referenceCount.get();
-
- if (finalReferenceCount == 1) {
- unpinPhysicallyIfOutOfTimeToLive();
- } else if (finalReferenceCount < 1) {
- throw new PipeRuntimeCriticalException(
- String.format(
- "wal %d is unpinned more than pinned, this should not happen",
- walEntryHandler.getMemTableId()));
- }
-
- referenceCount.decrementAndGet();
- }
-
- protected abstract void unpinInternal()
- throws MemTablePinException, PipeRuntimeNonCriticalException;
-
- /**
- * Invalidate the wal if it is unpinned and out of time to live.
- *
- * @return true if the wal is invalidated, false otherwise
- */
- public final boolean invalidateIfPossible() {
- if (referenceCount.get() > 0) {
- return false;
- }
-
- // referenceCount.get() == 0
- return unpinPhysicallyIfOutOfTimeToLive();
- }
-
- /**
- * Unpin the wal if it is out of time to live.
- *
- * @return true if the wal is unpinned physically (then it can be invalidated), false otherwise
- * @throws PipeRuntimeNonCriticalException if failed to unpin WAL of memtable.
- */
- private boolean unpinPhysicallyIfOutOfTimeToLive() {
- if (isPhysicallyPinned.get()) {
- if (System.currentTimeMillis() - lastLogicalPinTime.get() > WAL_MIN_TIME_TO_LIVE_IN_MS) {
- try {
- unpinInternal();
- } catch (MemTablePinException e) {
- throw new PipeRuntimeNonCriticalException(
- String.format(
- "failed to unpin wal %d, because %s",
- walEntryHandler.getMemTableId(), e.getMessage()));
- }
- isPhysicallyPinned.set(false);
- LOGGER.info(
- "wal {} is unpinned by pipe engine when checking time to live",
- walEntryHandler.getMemTableId());
- return true;
- } else {
- return false;
- }
- } else {
- LOGGER.info(
- "wal {} is not pinned physically when checking time to live",
- walEntryHandler.getMemTableId());
- return true;
- }
- }
-
- @Override
- public final void close() {
- if (isPhysicallyPinned.get()) {
- try {
- unpinInternal();
- } catch (MemTablePinException e) {
- LOGGER.error(
- "failed to unpin wal {} when closing pipe wal resource, because {}",
- walEntryHandler.getMemTableId(),
- e.getMessage());
- }
- isPhysicallyPinned.set(false);
- LOGGER.info(
- "wal {} is unpinned by pipe engine when closing pipe wal resource",
- walEntryHandler.getMemTableId());
- }
-
- referenceCount.set(0);
- }
-
- public int getReferenceCount() {
- return referenceCount.get();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
deleted file mode 100644
index 9c51d79daad..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ConcurrentModificationException;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantLock;
-
-public abstract class PipeWALResourceManager {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PipeWALResourceManager.class);
-
- protected final Map<Long, PipeWALResource> memtableIdToPipeWALResourceMap;
-
- private static final int SEGMENT_LOCK_COUNT = 32;
- private final ReentrantLock[] memTableIdSegmentLocks;
-
- protected PipeWALResourceManager() {
- // memTableIdToPipeWALResourceMap can be concurrently accessed by multiple threads
- memtableIdToPipeWALResourceMap = new ConcurrentHashMap<>();
-
- memTableIdSegmentLocks = new ReentrantLock[SEGMENT_LOCK_COUNT];
- for (int i = 0; i < SEGMENT_LOCK_COUNT; i++) {
- memTableIdSegmentLocks[i] = new ReentrantLock();
- }
-
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "PipeWALResourceManager#ttlCheck()",
- this::ttlCheck,
- Math.max(PipeWALResource.WAL_MIN_TIME_TO_LIVE_IN_MS / 1000, 1));
- }
-
- @SuppressWarnings("java:S2222")
- private void ttlCheck() {
- final Iterator<Map.Entry<Long, PipeWALResource>> iterator =
- memtableIdToPipeWALResourceMap.entrySet().iterator();
- final Optional<Logger> logger =
- PipeDataNodeResourceManager.log()
- .schedule(
- PipeWALResourceManager.class,
- PipeConfig.getInstance().getPipeWalPinMaxLogNumPerRound(),
- PipeConfig.getInstance().getPipeWalPinMaxLogIntervalRounds(),
- memtableIdToPipeWALResourceMap.size());
-
- final StringBuilder logBuilder = new StringBuilder();
- try {
- while (iterator.hasNext()) {
- final Map.Entry<Long, PipeWALResource> entry = iterator.next();
- final ReentrantLock lock =
- memTableIdSegmentLocks[(int) (entry.getKey() % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- if (entry.getValue().invalidateIfPossible()) {
- iterator.remove();
- } else {
- logBuilder.append(
- String.format(
- "<%d , %d times> ", entry.getKey(), entry.getValue().getReferenceCount()));
- }
- } finally {
- lock.unlock();
- }
- }
- } catch (final ConcurrentModificationException e) {
- LOGGER.error(
- "Concurrent modification issues happened, skipping the WAL in this round of ttl check",
- e);
- } finally {
- if (logBuilder.length() > 0) {
- logger.ifPresent(l -> l.info("WAL {}are still referenced", logBuilder));
- }
- }
- }
-
- public final void pin(final WALEntryHandler walEntryHandler) throws IOException {
- final long memTableId = walEntryHandler.getMemTableId();
- final ReentrantLock lock = memTableIdSegmentLocks[(int) (memTableId % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- pinInternal(memTableId, walEntryHandler);
- } finally {
- lock.unlock();
- }
- }
-
- protected abstract void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler)
- throws IOException;
-
- public final void unpin(final WALEntryHandler walEntryHandler) throws IOException {
- final long memTableId = walEntryHandler.getMemTableId();
- final ReentrantLock lock = memTableIdSegmentLocks[(int) (memTableId % SEGMENT_LOCK_COUNT)];
-
- lock.lock();
- try {
- unpinInternal(memTableId, walEntryHandler);
- } finally {
- lock.unlock();
- }
- }
-
- protected abstract void unpinInternal(
- final long memTableId, final WALEntryHandler walEntryHandler) throws IOException;
-
- public int getPinnedWalCount() {
- return Objects.nonNull(memtableIdToPipeWALResourceMap)
- ? memtableIdToPipeWALResourceMap.size()
- : 0;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
deleted file mode 100644
index f1ad513ccc4..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResource.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.hardlink;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALHardlinkResource extends PipeWALResource {
-
- private final PipeWALHardlinkResourceManager resourceManager;
-
- protected PipeWALHardlinkResource(
- WALEntryHandler walEntryHandler, PipeWALHardlinkResourceManager resourceManager) {
- super(walEntryHandler);
- this.resourceManager = resourceManager;
- }
-
- @Override
- protected void pinInternal() throws MemTablePinException {
- // TODO: hardlink
- walEntryHandler.pinMemTable();
- }
-
- @Override
- protected void unpinInternal() throws MemTablePinException {
- // TODO: hardlink
- walEntryHandler.unpinMemTable();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
deleted file mode 100644
index eebf766dc36..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.hardlink;
-
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.utils.FileUtils;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.HashMap;
-import java.util.Map;
-
-public class PipeWALHardlinkResourceManager extends PipeWALResourceManager {
-
- @Override
- protected void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap
- .computeIfAbsent(memTableId, id -> new PipeWALHardlinkResource(walEntryHandler, this))
- .pin();
- }
-
- @Override
- protected void unpinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap.get(memTableId).unpin();
- }
-
- //////////////////////////// hardlink related ////////////////////////////
-
- private final Map<String, Integer> hardlinkToReferenceMap = new HashMap<>();
-
- /**
- * given a file, create a hardlink, maintain a reference count for the hardlink, and return the
- * hardlink.
- *
- * <p>if the given file is already a hardlink, increase its reference count and return it.
- *
- * <p>if the given file is a wal, create a hardlink in pipe dir, increase the reference count of
- * the hardlink and return it.
- *
- * @param file wal file. can be original file or the hardlink of original file
- * @return the hardlink
- * @throws IOException when create hardlink failed
- */
- public synchronized File increaseFileReference(final File file) throws IOException {
- // if the file is already a hardlink, just increase reference count and return it
- if (increaseReferenceIfExists(file.getPath())) {
- return file;
- }
-
- // if the file is not a hardlink, check if there is a related hardlink in pipe dir. if so,
- // increase reference count and return it.
- final File hardlink = getHardlinkInPipeWALDir(file);
- if (increaseReferenceIfExists(hardlink.getPath())) {
- return hardlink;
- }
-
- // if the file is a wal, and there is no related hardlink in pipe dir, create a hardlink to pipe
- // dir, maintain a reference count for the hardlink, and return the hardlink.
- hardlinkToReferenceMap.put(hardlink.getPath(), 1);
- return FileUtils.createHardLink(file, hardlink);
- }
-
- private boolean increaseReferenceIfExists(final String path) {
- hardlinkToReferenceMap.computeIfPresent(path, (k, v) -> v + 1);
- return hardlinkToReferenceMap.containsKey(path);
- }
-
- // TODO: Check me! Make sure the file is not a hardlink.
- // TODO: IF user specify a wal by config, will the method work?
- private static File getHardlinkInPipeWALDir(final File file) throws IOException {
- try {
- return new File(getPipeWALDirPath(file), getRelativeFilePath(file));
- } catch (final Exception e) {
- throw new IOException(
- String.format(
- "failed to get hardlink in pipe dir " + "for file %s, it is not a wal",
- file.getPath()),
- e);
- }
- }
-
- private static String getPipeWALDirPath(File file) throws IOException {
- while (!file.getName().equals(IoTDBConstant.WAL_FOLDER_NAME)) {
- file = file.getParentFile();
- }
-
- return file.getParentFile().getCanonicalPath()
- + File.separator
- + IoTDBConstant.DATA_FOLDER_NAME
- + File.separator
- + PipeConfig.getInstance().getPipeHardlinkBaseDirName()
- + File.separator
- + PipeConfig.getInstance().getPipeHardlinkWALDirName();
- }
-
- private static String getRelativeFilePath(File file) {
- StringBuilder builder = new StringBuilder(file.getName());
- while (!file.getParentFile().getName().equals(IoTDBConstant.WAL_FOLDER_NAME)) {
- file = file.getParentFile();
- builder =
- new StringBuilder(file.getName())
- .append(IoTDBConstant.FILE_NAME_SEPARATOR)
- .append(builder);
- }
- return builder.toString();
- }
-
- /**
- * given a hardlink, decrease its reference count, if the reference count is 0, delete the file.
- * if the given file is not a hardlink, do nothing.
- *
- * @param hardlink the hardlinked file
- * @throws IOException when delete file failed
- */
- public synchronized void decreaseFileReference(final File hardlink) throws IOException {
- final Integer updatedReference =
- hardlinkToReferenceMap.computeIfPresent(
- hardlink.getPath(), (file, reference) -> reference - 1);
-
- if (updatedReference != null && updatedReference == 0) {
- Files.deleteIfExists(hardlink.toPath());
- hardlinkToReferenceMap.remove(hardlink.getPath());
- }
- }
-
- @TestOnly
- public synchronized int getFileReferenceCount(final File hardlink) {
- return hardlinkToReferenceMap.getOrDefault(hardlink.getPath(), 0);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
deleted file mode 100644
index e8e03e64a16..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResource.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.selfhost;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALSelfHostResource extends PipeWALResource {
-
- public PipeWALSelfHostResource(WALEntryHandler walEntryHandler) {
- super(walEntryHandler);
- }
-
- @Override
- protected void pinInternal() throws MemTablePinException {
- walEntryHandler.pinMemTable();
- }
-
- @Override
- protected void unpinInternal() throws MemTablePinException {
- walEntryHandler.unpinMemTable();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
deleted file mode 100644
index c7fe0accda2..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/selfhost/PipeWALSelfHostResourceManager.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.pipe.resource.wal.selfhost;
-
-import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
-public class PipeWALSelfHostResourceManager extends PipeWALResourceManager {
-
- @Override
- protected void pinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap
- .computeIfAbsent(memTableId, id -> new PipeWALSelfHostResource(walEntryHandler))
- .pin();
- }
-
- @Override
- protected void unpinInternal(final long memTableId, final WALEntryHandler walEntryHandler) {
- memtableIdToPipeWALResourceMap.get(memTableId).unpin();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 226b26093ed..711ab737951 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -41,6 +41,9 @@ import org.apache.iotdb.common.rpc.thrift.TTestConnectionResult;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.request.AsyncRequestContext;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.ConfigurationFileUtils;
@@ -312,12 +315,18 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -364,6 +373,18 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig();
+ private final ExecutorService schemaExecutor =
+ new WrappedThreadPoolExecutor(
+ 0,
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount(),
+ 0L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(
+ IoTDBDescriptor.getInstance().getConfig().getSchemaThreadCount()),
+ new IoTThreadFactory(ThreadName.SCHEMA_PARALLEL_POOL.getName()),
+ ThreadName.SCHEMA_PARALLEL_POOL.getName(),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+
private static final String SYSTEM = "system";
public DataNodeInternalRPCServiceImpl() {
@@ -1152,6 +1173,10 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
.map(PipeMeta::deserialize4TaskAgent)
.collect(Collectors.toList()));
+ if (Objects.isNull(exceptionMessages)) {
+ return new TPushPipeMetaResp()
+ .setStatus(new TSStatus(TSStatusCode.PIPE_PUSH_META_TIMEOUT.getStatusCode()));
+ }
return exceptionMessages.isEmpty()
? new TPushPipeMetaResp()
.setStatus(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))
@@ -1428,16 +1453,31 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
final List<TSStatus> statusList = Collections.synchronizedList(new ArrayList<>());
final AtomicBoolean hasFailure = new AtomicBoolean(false);
- consensusGroupIdList.parallelStream()
- .forEach(
- consensusGroupId -> {
- final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
- && status.getCode() != TSStatusCode.ONLY_LOGICAL_VIEW.getStatusCode()) {
- hasFailure.set(true);
- }
- statusList.add(status);
- });
+ final Set<Future<?>> schemaFuture = new HashSet<>();
+
+ consensusGroupIdList.forEach(
+ consensusGroupId ->
+ schemaFuture.add(
+ schemaExecutor.submit(
+ () -> {
+ final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && status.getCode() != TSStatusCode.ONLY_LOGICAL_VIEW.getStatusCode()) {
+ hasFailure.set(true);
+ }
+ statusList.add(status);
+ })));
+
+ for (final Future<?> future : schemaFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing internal schema task: ", e);
+ statusList.add(
+ new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ .setMessage(e.toString()));
+ }
+ }
if (hasFailure.get()) {
return RpcUtils.getStatus(statusList);
@@ -1456,15 +1496,30 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
final List<TSStatus> statusList = Collections.synchronizedList(new ArrayList<>());
final AtomicBoolean hasFailure = new AtomicBoolean(false);
- consensusGroupIdList.parallelStream()
- .forEach(
- consensusGroupId -> {
- final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- hasFailure.set(true);
- }
- statusList.add(status);
- });
+ final Set<Future<?>> schemaFuture = new HashSet<>();
+
+ consensusGroupIdList.forEach(
+ consensusGroupId ->
+ schemaFuture.add(
+ schemaExecutor.submit(
+ () -> {
+ final TSStatus status = executeOnOneRegion.apply(consensusGroupId);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ hasFailure.set(true);
+ }
+ statusList.add(status);
+ })));
+
+ for (final Future<?> future : schemaFuture) {
+ try {
+ future.get();
+ } catch (final ExecutionException | InterruptedException e) {
+ LOGGER.warn("Exception occurs when executing internal schema task: ", e);
+ statusList.add(
+ new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ .setMessage(e.toString()));
+ }
+ }
if (hasFailure.get()) {
return RpcUtils.getStatus(statusList);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
index c1d8c3fe979..f9a02c56799 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
@@ -464,7 +464,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
final long startTime = System.nanoTime();
try {
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
- new LoadTsFileDataTypeConverter(isGeneratedByPipe);
+ new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
final TSStatus status =
isTableModelTsFile.get(i)
@@ -722,7 +722,7 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
- new LoadTsFileDataTypeConverter(isGeneratedByPipe);
+ new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
for (int i = 0; i < tsFiles.size(); i++) {
final long startTime = System.nanoTime();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
index 6a0532a9b50..7709e20b557 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
@@ -566,7 +566,7 @@ public class LoadTsFileScheduler implements IScheduler {
private void convertFailedTsFilesToTabletsAndRetry() {
final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
- new LoadTsFileDataTypeConverter(isGeneratedByPipe);
+ new LoadTsFileDataTypeConverter(queryContext, isGeneratedByPipe);
final Iterator<Integer> iterator = failedTsFileNodeIndexes.listIterator();
while (iterator.hasNext()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index 2c2281dcf66..fb276150956 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -38,7 +38,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.ShutdownException;
import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.commons.file.SystemFileFactory;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.schema.ttl.TTLCache;
import org.apache.iotdb.commons.service.IService;
import org.apache.iotdb.commons.service.ServiceType;
@@ -219,12 +218,6 @@ public class StorageEngine implements IService {
LOGGER.info(
"Storage Engine recover cost: {}s.",
(System.currentTimeMillis() - startRecoverTime) / 1000);
-
- PipeDataNodeAgent.runtime()
- .registerPeriodicalJob(
- "StorageEngine#operateFlush",
- () -> operateFlush(new TFlushReq()),
- PipeConfig.getInstance().getPipeStorageEngineFlushTimeIntervalMs() / 1000);
},
ThreadName.STORAGE_ENGINE_RECOVER_TRIGGER.getName());
recoverEndTrigger.start();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 6a3b0917422..f4f4b9c3988 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1996,7 +1996,7 @@ public class DataRegion implements IDataRegionForQuery {
}
/** close all working tsfile processors */
- private List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
+ public List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
writeLock("asyncCloseAllWorkingTsFileProcessors");
List<Future<?>> futures = new ArrayList<>();
int count = 0;
@@ -3358,7 +3358,7 @@ public class DataRegion implements IDataRegionForQuery {
// Listen before the tsFile is added into tsFile manager to avoid it being compacted
PipeInsertionDataNodeListener.getInstance()
- .listenToTsFile(dataRegionId, databaseName, tsFileResource, true, isGeneratedByPipe);
+ .listenToTsFile(dataRegionId, databaseName, tsFileResource, true);
tsFileManager.add(tsFileResource, false);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index 6c80053994e..c2aee2bde90 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -225,7 +225,7 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask {
targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
targetFile.deserialize();
- targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
+ targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndex());
}
private boolean recoverTaskInfoFromLogFile() throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index 44905afc461..ee7afe8f88c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -329,8 +329,11 @@ public class CompactionUtils {
List<TsFileResource> seqResources,
List<TsFileResource> unseqResources) {
for (TsFileResource targetResource : targetResources) {
+ // Initial value
+ targetResource.setGeneratedByPipe(true);
+ targetResource.setGeneratedByPipeConsensus(true);
for (TsFileResource unseqResource : unseqResources) {
- targetResource.updateProgressIndex(unseqResource.getMaxProgressIndexAfterClose());
+ targetResource.updateProgressIndex(unseqResource.getMaxProgressIndex());
targetResource.setGeneratedByPipe(
unseqResource.isGeneratedByPipe() && targetResource.isGeneratedByPipe());
targetResource.setGeneratedByPipeConsensus(
@@ -338,7 +341,7 @@ public class CompactionUtils {
&& targetResource.isGeneratedByPipeConsensus());
}
for (TsFileResource seqResource : seqResources) {
- targetResource.updateProgressIndex(seqResource.getMaxProgressIndexAfterClose());
+ targetResource.updateProgressIndex(seqResource.getMaxProgressIndex());
targetResource.setGeneratedByPipe(
seqResource.isGeneratedByPipe() && targetResource.isGeneratedByPipe());
targetResource.setGeneratedByPipeConsensus(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java
index 9b730b7e300..3eac19188e9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/NotifyFlushMemTable.java
@@ -36,11 +36,4 @@ public class NotifyFlushMemTable extends AbstractMemTable {
public boolean isSignalMemTable() {
return true;
}
-
- @Override
- public boolean isTotallyGeneratedByPipe() {
- // Even though the `isTotallyGeneratedByPipe` for the corresponding memory table of this
- // `NotifyFlushMemTable` might be true, we still return false to ensure data integrity.
- return false;
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index d550781858f..a4b235650cf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -68,7 +68,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -122,8 +121,6 @@ public abstract class AbstractMemTable implements IMemTable {
private String database;
private String dataRegionId;
- private final AtomicBoolean isTotallyGeneratedByPipe = new AtomicBoolean(true);
-
protected AbstractMemTable() {
this.database = null;
this.dataRegionId = null;
@@ -1012,14 +1009,4 @@ public abstract class AbstractMemTable implements IMemTable {
this.database = database;
this.dataRegionId = dataRegionId;
}
-
- @Override
- public void markAsNotGeneratedByPipe() {
- this.isTotallyGeneratedByPipe.set(false);
- }
-
- @Override
- public boolean isTotallyGeneratedByPipe() {
- return this.isTotallyGeneratedByPipe.get();
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
index 44098b69a56..6c6e09c5782 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
@@ -198,8 +198,4 @@ public interface IMemTable extends WALEntryValue {
String getDataRegionId();
void setDatabaseAndDataRegionId(String database, String dataRegionId);
-
- void markAsNotGeneratedByPipe();
-
- boolean isTotallyGeneratedByPipe();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index b2c96a87f3b..cdf66b2da3c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -110,6 +110,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
@@ -187,6 +188,8 @@ public class TsFileProcessor {
/** Total memtable size for mem control. */
private long totalMemTableSize;
+ private final AtomicBoolean isTotallyGeneratedByPipe = new AtomicBoolean(true);
+
private static final String FLUSH_QUERY_WRITE_LOCKED = "{}: {} get flushQueryLock write lock";
private static final String FLUSH_QUERY_WRITE_RELEASE =
"{}: {} get flushQueryLock write lock released";
@@ -326,13 +329,12 @@ public class TsFileProcessor {
PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowNode);
if (!insertRowNode.isGeneratedByPipe()) {
- workMemTable.markAsNotGeneratedByPipe();
+ this.isTotallyGeneratedByPipe.set(false);
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
dataRegionInfo.getDataRegion().getDataRegionId(),
dataRegionInfo.getDataRegion().getDatabaseName(),
- walFlushListener.getWalEntryHandler(),
insertRowNode,
tsFileResource);
@@ -423,13 +425,12 @@ public class TsFileProcessor {
PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertRowsNode);
if (!insertRowsNode.isGeneratedByPipe()) {
- workMemTable.markAsNotGeneratedByPipe();
+ this.isTotallyGeneratedByPipe.set(false);
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
dataRegionInfo.getDataRegion().getDataRegionId(),
dataRegionInfo.getDataRegion().getDatabaseName(),
- walFlushListener.getWalEntryHandler(),
insertRowsNode,
tsFileResource);
@@ -596,13 +597,12 @@ public class TsFileProcessor {
PipeDataNodeAgent.runtime().assignSimpleProgressIndexIfNeeded(insertTabletNode);
if (!insertTabletNode.isGeneratedByPipe()) {
- workMemTable.markAsNotGeneratedByPipe();
+ this.isTotallyGeneratedByPipe.set(false);
}
PipeInsertionDataNodeListener.getInstance()
.listenToInsertNode(
dataRegionInfo.getDataRegion().getDataRegionId(),
dataRegionInfo.getDataRegion().getDatabaseName(),
- walFlushListener.getWalEntryHandler(),
insertTabletNode,
tsFileResource);
@@ -1299,6 +1299,7 @@ public class TsFileProcessor {
// we have to add the memtable into flushingList first and then set the shouldClose tag.
// see https://issues.apache.org/jira/browse/IOTDB-510
IMemTable tmpMemTable = workMemTable == null ? new NotifyFlushMemTable() : workMemTable;
+ tsFileResource.setGeneratedByPipe(isTotallyGeneratedByPipe.get());
try {
PipeInsertionDataNodeListener.getInstance()
@@ -1306,8 +1307,7 @@ public class TsFileProcessor {
dataRegionInfo.getDataRegion().getDataRegionId(),
dataRegionInfo.getDataRegion().getDatabaseName(),
tsFileResource,
- false,
- tmpMemTable.isTotallyGeneratedByPipe());
+ false);
// When invoke closing TsFile after insert data to memTable, we shouldn't flush until invoke
// flushing memTable in System module.
@@ -1739,6 +1739,16 @@ public class TsFileProcessor {
logger.debug("Start to end file {}", tsFileResource);
}
writer.endFile();
+
+ // Listen after "endFile" to avoid unnecessary waiting for tsFile close
+ // before resource serialization to avoid missing hardlink after restart
+ PipeInsertionDataNodeListener.getInstance()
+ .listenToTsFile(
+ dataRegionInfo.getDataRegion().getDataRegionId(),
+ dataRegionInfo.getDataRegion().getDatabaseName(),
+ tsFileResource,
+ false);
+
tsFileResource.serialize();
FileTimeIndexCacheRecorder.getInstance().logFileTimeIndex(tsFileResource);
if (logger.isDebugEnabled()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
index 94cc87b97ca..15c2365ebdb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.dataregion.tsfile;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModFileManagement;
import org.apache.iotdb.db.storageengine.dataregion.modification.PartitionLevelModFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndexCacheRecorder;
@@ -299,6 +300,18 @@ public class TsFileManager {
} finally {
writeUnlock();
}
+
+ // Currently disable
+ if (false) {
+ PipeDataNodeResourceManager.compaction()
+ .emitResult(
+ storageGroupName,
+ dataRegionId,
+ timePartition,
+ seqFileResources,
+ unseqFileResources,
+ targetFileResources);
+ }
}
public boolean contains(TsFileResource tsFileResource, boolean sequence) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 59452f0696f..44509e0fe7f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.load.PartitionViolationException;
-import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper;
import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils.InsertionCompactionCandidateStatus;
@@ -195,13 +194,13 @@ public class TsFileResource implements PersistentResource {
*/
private TsFileResource originTsFileResource;
- private ProgressIndex maxProgressIndex;
+ private final AtomicReference<ProgressIndex> maxProgressIndex = new AtomicReference<>();
/** used to prevent circular replication in PipeConsensus */
- private boolean isGeneratedByPipeConsensus = false;
+ private volatile boolean isGeneratedByPipeConsensus = false;
/** used to prevent circular replication in Pipe */
- private boolean isGeneratedByPipe = false;
+ private volatile boolean isGeneratedByPipe = false;
private InsertionCompactionCandidateStatus insertionCompactionCandidateStatus =
InsertionCompactionCandidateStatus.NOT_CHECKED;
@@ -302,9 +301,9 @@ public class TsFileResource implements PersistentResource {
ReadWriteIOUtils.write((String) null, outputStream);
}
- if (maxProgressIndex != null) {
+ if (maxProgressIndex.get() != null) {
TsFileResourceBlockType.PROGRESS_INDEX.serialize(outputStream);
- maxProgressIndex.serialize(outputStream);
+ maxProgressIndex.get().serialize(outputStream);
} else {
TsFileResourceBlockType.EMPTY_BLOCK.serialize(outputStream);
}
@@ -341,7 +340,7 @@ public class TsFileResource implements PersistentResource {
TsFileResourceBlockType.deserialize(ReadWriteIOUtils.readByte(inputStream));
switch (blockType) {
case PROGRESS_INDEX:
- maxProgressIndex = ProgressIndexType.deserializeFrom(inputStream);
+ maxProgressIndex.set(ProgressIndexType.deserializeFrom(inputStream));
break;
case PIPE_MARK:
isGeneratedByPipeConsensus = ReadWriteIOUtils.readBoolean(inputStream);
@@ -1387,13 +1386,9 @@ public class TsFileResource implements PersistentResource {
return;
}
- maxProgressIndex =
- (maxProgressIndex == null
- ? progressIndex
- : maxProgressIndex.updateToMinimumEqualOrIsAfterProgressIndex(progressIndex));
-
- PipeTsFileEpochProgressIndexKeeper.getInstance()
- .updateProgressIndex(getDataRegionId(), getTsFilePath(), maxProgressIndex);
+ if (!maxProgressIndex.compareAndSet(null, progressIndex)) {
+ maxProgressIndex.get().updateToMinimumEqualOrIsAfterProgressIndex(progressIndex);
+ }
}
public void setProgressIndex(ProgressIndex progressIndex) {
@@ -1401,10 +1396,7 @@ public class TsFileResource implements PersistentResource {
return;
}
- maxProgressIndex = progressIndex;
-
- PipeTsFileEpochProgressIndexKeeper.getInstance()
- .updateProgressIndex(getDataRegionId(), getTsFilePath(), maxProgressIndex);
+ maxProgressIndex.set(progressIndex);
}
@Override
@@ -1421,7 +1413,8 @@ public class TsFileResource implements PersistentResource {
}
public ProgressIndex getMaxProgressIndex() {
- return maxProgressIndex == null ? MinimumProgressIndex.INSTANCE : maxProgressIndex;
+ final ProgressIndex index = maxProgressIndex.get();
+ return index == null ? MinimumProgressIndex.INSTANCE : index;
}
public boolean isEmpty() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index 16be3f8ad08..8cc080016ca 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -345,7 +345,6 @@ public class WALBuffer extends AbstractWALBuffer {
info.metaData.add(size, searchIndex, walEntry.getMemTableId());
info.memTableId2WalDiskUsage.compute(
walEntry.getMemTableId(), (k, v) -> v == null ? size : v + size);
- walEntry.getWalFlushListener().getWalEntryHandler().setSize(size);
info.fsyncListeners.add(walEntry.getWalFlushListener());
}
@@ -606,13 +605,8 @@ public class WALBuffer extends AbstractWALBuffer {
// notify all waiting listeners
if (forceSuccess) {
- long position = lastFsyncPosition;
for (WALFlushListener fsyncListener : info.fsyncListeners) {
fsyncListener.succeed();
- if (fsyncListener.getWalEntryHandler() != null) {
- fsyncListener.getWalEntryHandler().setEntryPosition(walFileVersionId, position);
- position += fsyncListener.getWalEntryHandler().getSize();
- }
}
lastFsyncPosition = currentWALFileWriter.originalSize();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
index 88cfb8b1564..f7b85089feb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
@@ -80,14 +80,14 @@ public abstract class WALEntry implements SerializedSize {
} else {
throw new RuntimeException("Unknown WALEntry type");
}
- walFlushListener = new WALFlushListener(wait, value);
+ walFlushListener = new WALFlushListener(wait);
}
protected WALEntry(WALEntryType type, long memTableId, WALEntryValue value, boolean wait) {
this.type = type;
this.memTableId = memTableId;
this.value = value;
- this.walFlushListener = new WALFlushListener(wait, value);
+ this.walFlushListener = new WALFlushListener(wait);
}
public abstract void serialize(IWALByteBufferView buffer);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
index afa6651dfa4..6f801a78457 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
@@ -25,11 +25,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.ILogWriter;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALInsertNodeCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,9 +175,7 @@ public class CheckpointManager implements AutoCloseable {
return;
}
memTableInfo.setFlushed();
- if (!memTableInfo.isPinned()) {
- memTableId2Info.remove(memTableId);
- }
+ memTableId2Info.remove(memTableId);
Checkpoint checkpoint =
new Checkpoint(
CheckpointType.FLUSH_MEMORY_TABLE, Collections.singletonList(memTableInfo));
@@ -261,71 +257,9 @@ public class CheckpointManager implements AutoCloseable {
// endregion
- // region methods for pipe
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable(long memTableId) throws MemTablePinException {
- infoLock.lock();
- try {
- if (!memTableId2Info.containsKey(memTableId)) {
- throw new MemTablePinException(
- String.format(
- "Fail to pin memTable-%d because this memTable doesn't exist in the wal.",
- memTableId));
- }
- MemTableInfo memTableInfo = memTableId2Info.get(memTableId);
- if (!memTableInfo.isPinned()) {
- WALInsertNodeCache.getInstance().addMemTable(memTableId);
- }
- memTableInfo.pin();
- } finally {
- infoLock.unlock();
- }
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable(long memTableId) throws MemTablePinException {
- infoLock.lock();
- try {
- if (!memTableId2Info.containsKey(memTableId)) {
- throw new MemTablePinException(
- String.format(
- "Fail to unpin memTable-%d because this memTable doesn't exist in the wal.",
- memTableId));
- }
- if (!memTableId2Info.get(memTableId).isPinned()) {
- throw new MemTablePinException(
- String.format(
- "Fail to unpin memTable-%d because this memTable hasn't been pinned.", memTableId));
- }
- MemTableInfo memTableInfo = memTableId2Info.get(memTableId);
- memTableInfo.unpin();
- if (!memTableInfo.isPinned()) {
- WALInsertNodeCache.getInstance().removeMemTable(memTableId);
- if (memTableInfo.isFlushed()) {
- memTableId2Info.remove(memTableId);
- }
- }
- } finally {
- infoLock.unlock();
- }
- }
-
- // endregion
-
- /** Get MemTableInfo of oldest unpinned MemTable, whose first version id is smallest. */
- public MemTableInfo getOldestUnpinnedMemTableInfo() {
+ public MemTableInfo getOldestMemTableInfo() {
// find oldest memTable
return activeOrPinnedMemTables().stream()
- .filter(memTableInfo -> !memTableInfo.isPinned())
.min(Comparator.comparingLong(MemTableInfo::getMemTableId))
.orElse(null);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
index 59c2a3be23b..984006b25bc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/MemTableInfo.java
@@ -47,8 +47,7 @@ public class MemTableInfo implements WALEntryValue {
// memTable
private IMemTable memTable;
- // memTable pin count
- private int pinCount;
+
// memTable is flushed or not
private boolean flushed;
// data region id
@@ -116,22 +115,6 @@ public class MemTableInfo implements WALEntryValue {
return memTable;
}
- public void pin() {
- this.pinCount++;
- }
-
- public void unpin() {
- this.pinCount--;
- }
-
- public boolean isPinned() {
- return pinCount > 0;
- }
-
- public int getPinCount() {
- return pinCount;
- }
-
public boolean isFlushed() {
return flushed;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
index b03b27a6994..2f257da9adc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.storageengine.dataregion.wal.io;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryPosition;
import java.io.Closeable;
import java.io.DataInputStream;
@@ -50,18 +49,6 @@ public class WALByteBufReader implements Closeable {
}
}
- public WALByteBufReader(WALEntryPosition walEntryPosition) throws IOException {
- WALInputStream walInputStream = walEntryPosition.openReadFileStream();
- try {
- this.logStream = new DataInputStream(walInputStream);
- this.metaData = walInputStream.getWALMetaData();
- this.sizeIterator = metaData.getBuffersSize().iterator();
- } catch (Exception e) {
- walInputStream.close();
- throw e;
- }
- }
-
/** Like {@link Iterator#hasNext()}. */
public boolean hasNext() {
return sizeIterator.hasNext();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
index c13e066415b..48bf4c1cfb9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
@@ -44,9 +44,9 @@ public class WALFakeNode implements IWALNode {
public WALFakeNode(Status status, Exception cause) {
this.status = status;
- this.successListener = new WALFlushListener(false, null);
+ this.successListener = new WALFlushListener(false);
this.successListener.succeed();
- this.failListener = new WALFlushListener(false, null);
+ this.failListener = new WALFlushListener(false);
this.failListener.fail(cause);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
index 69af5d62162..8803aca56c4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
@@ -50,7 +50,6 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointType;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.MemTableInfo;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALByteBufReader;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
@@ -58,7 +57,6 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.AbstractR
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.AbstractResultListener.Status;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALFlushListener;
-import org.apache.commons.lang3.StringUtils;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.utils.TsFileUtils;
import org.slf4j.Logger;
@@ -196,7 +194,6 @@ public class WALNode implements IWALNode {
buffer.write(walEntry);
// set handler for pipe
- walEntry.getWalFlushListener().getWalEntryHandler().setWalNode(this, walEntry.getMemTableId());
return walEntry.getWalFlushListener();
}
@@ -241,25 +238,6 @@ public class WALNode implements IWALNode {
// region methods for pipe
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable(long memTableId) throws MemTablePinException {
- checkpointManager.pinMemTable(memTableId);
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable(long memTableId) throws MemTablePinException {
- checkpointManager.unpinMemTable(memTableId);
- }
-
// endregion
// region Task to delete outdated .wal files
@@ -283,8 +261,6 @@ public class WALNode implements IWALNode {
// the effective information ratio
private double effectiveInfoRatio = 1.0d;
- private List<Long> pinnedMemTableIds;
-
private int fileIndexAfterFilterSafelyDeleteIndex = Integer.MAX_VALUE;
private List<Long> successfullyDeleted;
private long deleteFileSize;
@@ -310,7 +286,6 @@ public class WALNode implements IWALNode {
this.sortedWalFilesExcludingLast =
Arrays.copyOfRange(allWalFilesOfOneNode, 0, allWalFilesOfOneNode.length - 1);
this.activeOrPinnedMemTables = checkpointManager.activeOrPinnedMemTables();
- this.pinnedMemTableIds = initPinnedMemTableIds();
this.fileIndexAfterFilterSafelyDeleteIndex = initFileIndexAfterFilterSafelyDeleteIndex();
this.successfullyDeleted = new ArrayList<>();
this.deleteFileSize = 0;
@@ -331,20 +306,6 @@ public class WALNode implements IWALNode {
}
}
- private List<Long> initPinnedMemTableIds() {
- List<MemTableInfo> memTableInfos = checkpointManager.activeOrPinnedMemTables();
- if (memTableInfos.isEmpty()) {
- return new ArrayList<>();
- }
- List<Long> pinnedIds = new ArrayList<>();
- for (MemTableInfo memTableInfo : memTableInfos) {
- if (memTableInfo.isFlushed() && memTableInfo.isPinned()) {
- pinnedIds.add(memTableInfo.getMemTableId());
- }
- }
- return pinnedIds;
- }
-
@Override
public void run() {
// The intent of the loop execution here is to try to get as many memTable flush or snapshot
@@ -378,7 +339,7 @@ public class WALNode implements IWALNode {
private void updateEffectiveInfoRationAndUpdateMetric() {
// calculate effective information ratio
long costOfActiveMemTables = checkpointManager.getTotalCostOfActiveMemTables();
- MemTableInfo oldestUnpinnedMemTableInfo = checkpointManager.getOldestUnpinnedMemTableInfo();
+ MemTableInfo oldestUnpinnedMemTableInfo = checkpointManager.getOldestMemTableInfo();
long avgFileSize =
getFileNum() != 0
? getDiskUsage() / getFileNum()
@@ -402,45 +363,10 @@ public class WALNode implements IWALNode {
}
private void summarizeExecuteResult() {
- if (!pinnedMemTableIds.isEmpty()
- || fileIndexAfterFilterSafelyDeleteIndex < sortedWalFilesExcludingLast.length) {
- if (logger.isDebugEnabled()) {
- StringBuilder summary =
- new StringBuilder(
- String.format(
- "wal node-%s delete outdated files summary:the range is: [%d,%d], delete successful is [%s], safely delete file index is: [%s].The following reasons influenced the result: %s",
- identifier,
- WALFileUtils.parseVersionId(sortedWalFilesExcludingLast[0].getName()),
- WALFileUtils.parseVersionId(
- sortedWalFilesExcludingLast[sortedWalFilesExcludingLast.length - 1]
- .getName()),
- StringUtils.join(successfullyDeleted, ","),
- fileIndexAfterFilterSafelyDeleteIndex,
- System.lineSeparator()));
-
- if (!pinnedMemTableIds.isEmpty()) {
- summary
- .append("- MemTable has been flushed but pinned by PIPE, the MemTableId list is : ")
- .append(StringUtils.join(pinnedMemTableIds, ","))
- .append(".")
- .append(System.lineSeparator());
- }
- if (fileIndexAfterFilterSafelyDeleteIndex < sortedWalFilesExcludingLast.length) {
- summary.append(
- String.format(
- "- The data in the wal file was not consumed by the consensus group,current search index is %d, safely delete index is %d",
- getCurrentSearchIndex(), safelyDeletedSearchIndex));
- }
- String summaryLog = summary.toString();
- logger.debug(summaryLog);
- }
-
- } else {
- logger.debug(
- "Successfully delete {} outdated wal files for wal node-{}",
- successfullyDeleted.size(),
- identifier);
- }
+ logger.debug(
+ "Successfully delete {} outdated wal files for wal node-{}",
+ successfullyDeleted.size(),
+ identifier);
}
/** Delete obsolete wal files while recording which succeeded or failed */
@@ -488,20 +414,10 @@ public class WALNode implements IWALNode {
return false;
}
// find oldest memTable
- MemTableInfo oldestMemTableInfo = checkpointManager.getOldestUnpinnedMemTableInfo();
+ MemTableInfo oldestMemTableInfo = checkpointManager.getOldestMemTableInfo();
if (oldestMemTableInfo == null) {
return false;
}
- if (oldestMemTableInfo.isPinned()) {
- logger.warn(
- "Pipe: Effective information ratio {} of wal node-{} is below wal min effective info ratio {}. But fail to delete memTable-{}'s wal files because they are pinned by the Pipe module. Pin count: {}.",
- effectiveInfoRatio,
- identifier,
- config.getWalMinEffectiveInfoRatio(),
- oldestMemTableInfo.getMemTableId(),
- oldestMemTableInfo.getPinCount());
- return false;
- }
IMemTable oldestMemTable = oldestMemTableInfo.getMemTable();
if (oldestMemTable == null) {
return false;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
deleted file mode 100644
index f5d7406f5a6..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryHandler.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.MemTablePinException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
-import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
-
-import org.apache.tsfile.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * This handler is used by the Pipe to find the corresponding {@link InsertNode}. Besides, it can
- * try to pin/unpin the {@link WALEntry}s by the memTable id.
- */
-public class WALEntryHandler {
-
- private static final Logger logger = LoggerFactory.getLogger(WALEntryHandler.class);
-
- private long memTableId = -1;
-
- // cached value, null after this value is flushed to wal successfully
- @SuppressWarnings("squid:S3077")
- private volatile WALEntryValue value;
-
- // wal entry's position in the wal, valid after the value is flushed to wal successfully
- // it's safe to use volatile here to make this reference thread-safe.
- @SuppressWarnings("squid:S3077")
- private final WALEntryPosition walEntryPosition = new WALEntryPosition();
-
- // wal node, null when wal is disabled
- private WALNode walNode = null;
-
- private volatile boolean isHardlink = false;
- private final AtomicReference<File> hardlinkFile = new AtomicReference<>();
-
- public WALEntryHandler(final WALEntryValue value) {
- this.value = value;
- }
-
- /**
- * Pin the wal files of the given memory table. Notice: cannot pin one memTable too long,
- * otherwise the wal disk usage may too large.
- *
- * @throws MemTablePinException If the memTable has been flushed
- */
- public void pinMemTable() throws MemTablePinException {
- if (walNode == null || memTableId < 0) {
- throw new MemTablePinException("Fail to pin memTable because of internal error.");
- }
- walNode.pinMemTable(memTableId);
- }
-
- /**
- * Unpin the wal files of the given memory table.
- *
- * @throws MemTablePinException If there aren't corresponding pin operations
- */
- public void unpinMemTable() throws MemTablePinException {
- if (walNode == null || memTableId < 0) {
- throw new MemTablePinException("Fail to pin memTable because of internal error.");
- }
- walNode.unpinMemTable(memTableId);
- }
-
- public InsertNode getInsertNodeViaCacheIfPossible() {
- try {
- final WALEntryValue finalValue = value;
- if (finalValue instanceof InsertNode) {
- return (InsertNode) finalValue;
- }
- final Pair<ByteBuffer, InsertNode> byteBufferInsertNodePair =
- walEntryPosition.getByteBufferOrInsertNodeIfPossible();
- return byteBufferInsertNodePair == null ? null : byteBufferInsertNodePair.getRight();
- } catch (final Exception e) {
- logger.warn("Fail to get insert node via cache. {}", this, e);
- throw e;
- }
- }
-
- /**
- * Get this handler's value.
- *
- * @throws WALPipeException when failing to get the value.
- */
- public InsertNode getInsertNode() throws WALPipeException {
- // return local cache
- final WALEntryValue res = value;
- if (res != null) {
- if (res instanceof InsertNode) {
- return (InsertNode) res;
- } else {
- throw new WALPipeException("Fail to get value because the entry type isn't InsertNode.");
- }
- }
-
- // wait until the position is ready
- while (!walEntryPosition.canRead()) {
- try {
- synchronized (this) {
- this.wait();
- }
- } catch (final InterruptedException e) {
- logger.warn("Interrupted when waiting for result.", e);
- Thread.currentThread().interrupt();
- }
- }
-
- final InsertNode node = isHardlink ? readFromHardlinkWALFile() : readFromOriginalWALFile();
- if (node == null) {
- throw new WALPipeException(
- String.format("Fail to get the wal value of the position %s.", walEntryPosition));
- }
- return node;
- }
-
- public ByteBuffer getByteBuffer() throws WALPipeException {
- // wait until the position is ready
- while (!walEntryPosition.canRead()) {
- try {
- synchronized (this) {
- this.wait();
- }
- } catch (InterruptedException e) {
- logger.warn("Interrupted when waiting for result.", e);
- Thread.currentThread().interrupt();
- }
- }
-
- final ByteBuffer buffer = readByteBufferFromWALFile();
- if (buffer == null) {
- throw new WALPipeException(
- String.format("Fail to get the wal value of the position %s.", walEntryPosition));
- }
- return buffer;
- }
-
- private InsertNode readFromOriginalWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readInsertNodeViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- private InsertNode readFromHardlinkWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readInsertNodeViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- private ByteBuffer readByteBufferFromWALFile() throws WALPipeException {
- try {
- return walEntryPosition.readByteBufferViaCacheAfterCanRead();
- } catch (Exception e) {
- throw new WALPipeException("Fail to get value because the file content isn't correct.", e);
- }
- }
-
- public void setWalNode(final WALNode walNode, final long memTableId) {
- this.walNode = walNode;
- this.memTableId = memTableId;
- walEntryPosition.setWalNode(walNode, memTableId);
- }
-
- public long getMemTableId() {
- return memTableId;
- }
-
- public void setEntryPosition(final long walFileVersionId, final long position) {
- this.walEntryPosition.setEntryPosition(walFileVersionId, position, value);
- this.value = null;
- synchronized (this) {
- this.notifyAll();
- }
- }
-
- public WALEntryPosition getWalEntryPosition() {
- return walEntryPosition;
- }
-
- public int getSize() {
- return walEntryPosition.getSize();
- }
-
- public void setSize(final int size) {
- this.walEntryPosition.setSize(size);
- }
-
- public void hardlinkTo(File hardlinkFile) {
- isHardlink = true;
- this.hardlinkFile.set(hardlinkFile);
- }
-
- @Override
- public String toString() {
- return "WALEntryHandler{"
- + "memTableId="
- + memTableId
- + ", value="
- + value
- + ", walEntryPosition="
- + walEntryPosition
- + '}';
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
deleted file mode 100644
index 06ced2c32fa..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALInputStream;
-import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
-
-import org.apache.tsfile.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.StandardOpenOption;
-import java.util.Objects;
-
-/**
- * This class uses the tuple(identifier, file, position) to denote the position of the wal entry,
- * and give some methods to read the content from the disk.
- */
-public class WALEntryPosition {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WALEntryPosition.class);
-
- private volatile String identifier = "";
- private volatile long walFileVersionId = -1;
- private volatile long position;
- private volatile int size;
- // wal node, null when wal is disabled
- private WALNode walNode = null;
- // wal file is not null when openReadFileChannel method has been called
- private File walFile = null;
- // cache for wal entry
- private WALInsertNodeCache cache = null;
-
- private static final String ENTRY_NOT_READY_MESSAGE = "This entry isn't ready for read.";
-
- public WALEntryPosition() {}
-
- public WALEntryPosition(String identifier, long walFileVersionId, long position, int size) {
- this.identifier = identifier;
- this.walFileVersionId = walFileVersionId;
- this.position = position;
- this.size = size;
- }
-
- /**
- * Try to read the wal entry directly from the cache. No need to check if the wal entry is ready
- * for read.
- */
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNodeIfPossible() {
- return cache.getByteBufferOrInsertNodeIfPossible(this);
- }
-
- /**
- * Read the wal entry and parse it to the InsertNode. Use LRU cache to accelerate read.
- *
- * @throws IOException failing to read.
- */
- public InsertNode readInsertNodeViaCacheAfterCanRead() throws IOException {
- if (!canRead()) {
- throw new IOException(ENTRY_NOT_READY_MESSAGE);
- }
- return cache.getInsertNode(this);
- }
-
- /**
- * Read the wal entry and get the raw bytebuffer. Use LRU cache to accelerate read.
- *
- * @throws IOException failing to read.
- */
- public ByteBuffer readByteBufferViaCacheAfterCanRead() throws IOException {
- if (!canRead()) {
- throw new IOException(ENTRY_NOT_READY_MESSAGE);
- }
- return cache.getByteBuffer(this);
- }
-
- /**
- * Read the byte buffer directly.
- *
- * @throws IOException failing to read.
- */
- ByteBuffer read() throws IOException {
- if (!canRead()) {
- throw new IOException("Target file hasn't been specified.");
- }
- // TODO: Reuse the file stream
- try (WALInputStream is = openReadFileStream()) {
- is.skipToGivenLogicalPosition(position);
- ByteBuffer buffer = ByteBuffer.allocate(size);
- is.read(buffer);
- return buffer;
- } catch (Exception e) {
- LOGGER.error(
- "Unexpected error when reading a wal entry from {}@{} with size {}",
- walFile,
- position,
- size,
- e);
- throw new IOException(e);
- }
- }
-
- /**
- * Open the read file channel for this wal entry, this method will retry automatically when the
- * file is sealed when opening the file channel.
- *
- * @throws IOException failing to open the file channel.
- */
- public FileChannel openReadFileChannel() throws IOException {
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } else {
- try {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } catch (IOException e) {
- // unsealed file may be renamed after sealed, so we should try again
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return FileChannel.open(walFile.toPath(), StandardOpenOption.READ);
- } else {
- throw e;
- }
- }
- }
- }
-
- public WALInputStream openReadFileStream() throws IOException {
- // TODO: Refactor this part of code
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } else {
- try {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } catch (IOException e) {
- // unsealed file may be renamed after sealed, so we should try again
- if (isInSealedFile()) {
- walFile = walNode.getWALFile(walFileVersionId);
- return new WALInputStream(walFile);
- } else {
- throw e;
- }
- }
- }
- }
-
- public File getWalFile() {
- return walFile;
- }
-
- /** Return true only when the tuple(file, position, size) is ready. */
- public boolean canRead() {
- return walFileVersionId >= 0;
- }
-
- /** Return true only when this wal file is sealed. */
- public boolean isInSealedFile() {
- if (walNode == null || !canRead()) {
- throw new RuntimeException(ENTRY_NOT_READY_MESSAGE);
- }
- return walFileVersionId < walNode.getCurrentWALFileVersion();
- }
-
- public void setWalNode(WALNode walNode, long memTableId) {
- this.walNode = walNode;
- identifier = walNode.getIdentifier();
- cache = WALInsertNodeCache.getInstance();
- }
-
- public String getIdentifier() {
- return identifier;
- }
-
- public void setEntryPosition(long walFileVersionId, long position, WALEntryValue value) {
- this.position = position;
- this.walFileVersionId = walFileVersionId;
- if (cache != null && value instanceof InsertNode) {
- cache.cacheInsertNodeIfNeeded(this, (InsertNode) value);
- }
- }
-
- public long getPosition() {
- return position;
- }
-
- public long getWalFileVersionId() {
- return walFileVersionId;
- }
-
- public void setSize(int size) {
- this.size = size;
- }
-
- public int getSize() {
- return size;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(identifier, walFileVersionId, position);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- WALEntryPosition that = (WALEntryPosition) o;
- return identifier.equals(that.identifier)
- && walFileVersionId == that.walFileVersionId
- && position == that.position;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
deleted file mode 100644
index 431e8d0ba51..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALInsertNodeCache.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.storageengine.dataregion.wal.utils;
-
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.db.conf.DataNodeMemoryConfig;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
-import org.apache.iotdb.db.pipe.resource.memory.InsertNodeMemoryEstimator;
-import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlockType;
-import org.apache.iotdb.db.pipe.resource.memory.PipeModelFixedMemoryBlock;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType;
-import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALByteBufReader;
-
-import com.github.benmanes.caffeine.cache.CacheLoader;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.LoadingCache;
-import com.github.benmanes.caffeine.cache.Weigher;
-import org.apache.tsfile.utils.Pair;
-import org.checkerframework.checker.nullness.qual.NonNull;
-import org.checkerframework.checker.nullness.qual.Nullable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/** This cache is used by {@link WALEntryPosition}. */
-public class WALInsertNodeCache {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WALInsertNodeCache.class);
- private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final DataNodeMemoryConfig MEMORY_CONFIG =
- IoTDBDescriptor.getInstance().getMemoryConfig();
- private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
-
- private static PipeModelFixedMemoryBlock walModelFixedMemory = null;
-
- // LRU cache, find Pair<ByteBuffer, InsertNode> by WALEntryPosition
- private final LoadingCache<WALEntryPosition, Pair<ByteBuffer, InsertNode>> lruCache;
-
- // ids of all pinned memTables
- private final Set<Long> memTablesNeedSearch = ConcurrentHashMap.newKeySet();
-
- private volatile boolean hasPipeRunning = false;
-
- private WALInsertNodeCache() {
- if (walModelFixedMemory == null) {
- init();
- }
-
- final long requestedAllocateSize =
- (long)
- (PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- * PIPE_CONFIG.getPipeDataStructureWalMemoryProportion());
-
- lruCache =
- Caffeine.newBuilder()
- .maximumWeight(requestedAllocateSize)
- .weigher(
- (Weigher<WALEntryPosition, Pair<ByteBuffer, InsertNode>>)
- (position, pair) -> {
- long weightInLong = 0L;
- if (pair.right != null) {
- weightInLong = InsertNodeMemoryEstimator.sizeOf(pair.right);
- } else {
- weightInLong = position.getSize();
- }
- if (weightInLong <= 0) {
- return Integer.MAX_VALUE;
- }
- final int weightInInt = (int) weightInLong;
- return weightInInt != weightInLong ? Integer.MAX_VALUE : weightInInt;
- })
- .recordStats()
- .build(new WALInsertNodeCacheLoader());
- }
-
- // please call this method at PipeLauncher
- public static void init() {
- if (walModelFixedMemory != null) {
- return;
- }
- try {
- // Allocate memory for the fixed memory block of WAL
- walModelFixedMemory =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(
- (long)
- (PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- * PIPE_CONFIG.getPipeDataStructureWalMemoryProportion()),
- PipeMemoryBlockType.WAL);
- } catch (Exception e) {
- LOGGER.error("Failed to initialize WAL model fixed memory block", e);
- walModelFixedMemory =
- PipeDataNodeResourceManager.memory()
- .forceAllocateForModelFixedMemoryBlock(0, PipeMemoryBlockType.WAL);
- }
- }
-
- /////////////////////////// Getter & Setter ///////////////////////////
-
- public InsertNode getInsertNode(final WALEntryPosition position) {
- final Pair<ByteBuffer, InsertNode> pair = getByteBufferOrInsertNode(position);
-
- if (pair.getRight() != null) {
- return pair.getRight();
- }
-
- if (pair.getLeft() == null) {
- throw new IllegalStateException();
- }
-
- try {
- // multi pipes may share the same wal entry, so we need to wrap the byte[] into
- // different ByteBuffer for each pipe
- final InsertNode insertNode = parse(ByteBuffer.wrap(pair.getLeft().array()));
- pair.setRight(insertNode);
- return insertNode;
- } catch (final Exception e) {
- LOGGER.error(
- "Parsing failed when recovering insertNode from wal, walFile:{}, position:{}, size:{}, exception:",
- position.getWalFile(),
- position.getPosition(),
- position.getSize(),
- e);
- throw e;
- }
- }
-
- private InsertNode parse(final ByteBuffer buffer) {
- final PlanNode node = WALEntry.deserializeForConsensus(buffer);
- if (node instanceof InsertNode) {
- return (InsertNode) node;
- } else {
- return null;
- }
- }
-
- public ByteBuffer getByteBuffer(final WALEntryPosition position) {
- Pair<ByteBuffer, InsertNode> pair = getByteBufferOrInsertNode(position);
-
- if (pair.getLeft() != null) {
- // multi pipes may share the same wal entry, so we need to wrap the byte[] into
- // different ByteBuffer for each pipe
- return ByteBuffer.wrap(pair.getLeft().array());
- }
-
- // forbid multi threads to invalidate and load the same entry
- synchronized (this) {
- lruCache.invalidate(position);
- pair = getByteBufferOrInsertNode(position);
- }
-
- if (pair.getLeft() == null) {
- throw new IllegalStateException();
- }
-
- return ByteBuffer.wrap(pair.getLeft().array());
- }
-
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNode(final WALEntryPosition position) {
- hasPipeRunning = true;
-
- final Pair<ByteBuffer, InsertNode> pair = lruCache.get(position);
-
- if (pair == null) {
- throw new IllegalStateException();
- }
-
- return pair;
- }
-
- public Pair<ByteBuffer, InsertNode> getByteBufferOrInsertNodeIfPossible(
- final WALEntryPosition position) {
- hasPipeRunning = true;
- return lruCache.getIfPresent(position);
- }
-
- public void cacheInsertNodeIfNeeded(
- final WALEntryPosition walEntryPosition, final InsertNode insertNode) {
- // reduce memory usage
- if (hasPipeRunning) {
- lruCache.put(walEntryPosition, new Pair<>(null, insertNode));
- }
- }
-
- //////////////////////////// APIs provided for metric framework ////////////////////////////
-
- public double getCacheHitRate() {
- return Objects.nonNull(lruCache) ? lruCache.stats().hitRate() : 0;
- }
-
- public double getCacheHitCount() {
- return Objects.nonNull(lruCache) ? lruCache.stats().hitCount() : 0;
- }
-
- public double getCacheRequestCount() {
- return Objects.nonNull(lruCache) ? lruCache.stats().requestCount() : 0;
- }
-
- /////////////////////////// MemTable ///////////////////////////
-
- public void addMemTable(final long memTableId) {
- memTablesNeedSearch.add(memTableId);
- }
-
- public void removeMemTable(final long memTableId) {
- memTablesNeedSearch.remove(memTableId);
- }
-
- /////////////////////////// Cache Loader ///////////////////////////
-
- class WALInsertNodeCacheLoader
- implements CacheLoader<WALEntryPosition, Pair<ByteBuffer, InsertNode>> {
-
- @Override
- public @Nullable Pair<ByteBuffer, InsertNode> load(@NonNull final WALEntryPosition key)
- throws Exception {
- return new Pair<>(key.read(), null);
- }
-
- /** Batch load all wal entries in the file when any one key is absent. */
- @Override
- public @NonNull Map<@NonNull WALEntryPosition, @NonNull Pair<ByteBuffer, InsertNode>> loadAll(
- @NonNull final Iterable<? extends @NonNull WALEntryPosition> walEntryPositions) {
- final Map<WALEntryPosition, Pair<ByteBuffer, InsertNode>> loadedEntries = new HashMap<>();
-
- for (final WALEntryPosition walEntryPosition : walEntryPositions) {
- if (loadedEntries.containsKey(walEntryPosition) || !walEntryPosition.canRead()) {
- continue;
- }
-
- final long walFileVersionId = walEntryPosition.getWalFileVersionId();
-
- // load one when wal file is not sealed
- if (!walEntryPosition.isInSealedFile()) {
- try {
- loadedEntries.put(walEntryPosition, load(walEntryPosition));
- } catch (final Exception e) {
- LOGGER.info(
- "Fail to cache wal entries from the wal file with version id {}",
- walFileVersionId,
- e);
- }
- continue;
- }
-
- // batch load when wal file is sealed
- long position = 0;
- try (final WALByteBufReader walByteBufReader = new WALByteBufReader(walEntryPosition)) {
- while (walByteBufReader.hasNext()) {
- // see WALInfoEntry#serialize, entry type + memtable id + plan node type
- final ByteBuffer buffer = walByteBufReader.next();
-
- final int size = buffer.capacity();
- final WALEntryType type = WALEntryType.valueOf(buffer.get());
- final long memTableId = buffer.getLong();
-
- if ((memTablesNeedSearch.contains(memTableId)
- || walEntryPosition.getPosition() == position)
- && type.needSearch()) {
- buffer.clear();
- loadedEntries.put(
- new WALEntryPosition(
- walEntryPosition.getIdentifier(), walFileVersionId, position, size),
- new Pair<>(buffer, null));
- }
-
- position += size;
- }
- } catch (final IOException e) {
- LOGGER.info(
- "Fail to cache wal entries from the wal file with version id {}",
- walFileVersionId,
- e);
- }
- }
-
- return loadedEntries;
- }
- }
-
- /////////////////////////// Singleton ///////////////////////////
-
- public static WALInsertNodeCache getInstance() {
- return InstanceHolder.INSTANCE;
- }
-
- private static class InstanceHolder {
-
- public static final WALInsertNodeCache INSTANCE = new WALInsertNodeCache();
-
- private InstanceHolder() {
- // forbidding instantiation
- }
- }
-
- /////////////////////////// Test Only ///////////////////////////
-
- @TestOnly
- boolean contains(WALEntryPosition position) {
- return lruCache.getIfPresent(position) != null;
- }
-
- @TestOnly
- public void clear() {
- lruCache.invalidateAll();
- memTablesNeedSearch.clear();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
index 8c84a0cb0fe..7896ea01dcf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/listener/WALFlushListener.java
@@ -19,20 +19,10 @@
package org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener;
-import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
-
/** This class helps judge whether wal is flushed to the storage device. */
public class WALFlushListener extends AbstractResultListener {
- // handler for pipeline, only exists when value is InsertNode
- private final WALEntryHandler walEntryHandler;
- public WALFlushListener(boolean wait, WALEntryValue value) {
+ public WALFlushListener(boolean wait) {
super(wait);
- walEntryHandler = new WALEntryHandler(value);
- }
-
- public WALEntryHandler getWalEntryHandler() {
- return walEntryHandler;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index a0cdebdf030..6d9b7fe2e1a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -532,6 +532,7 @@ public class LoadTsFileManager {
final DataRegion dataRegion = entry.getKey().getDataRegion();
final TsFileResource tsFileResource = dataPartition2Resource.get(entry.getKey());
+ tsFileResource.setGeneratedByPipe(isGeneratedByPipe);
endTsFileResource(
writer,
tsFileResource,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java
index af3e371c9ed..08289df4808 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java
@@ -38,14 +38,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import static org.apache.iotdb.db.storageengine.load.converter.LoadTreeStatementDataTypeConvertExecutionVisitor.getExecutorPool;
+import static org.apache.iotdb.db.storageengine.load.converter.LoadTreeStatementDataTypeConvertExecutionVisitor.handleTSStatus;
public class LoadTableStatementDataTypeConvertExecutionVisitor
extends AstVisitor<Optional<TSStatus>, String> {
@@ -77,7 +73,6 @@ public class LoadTableStatementDataTypeConvertExecutionVisitor
LOGGER.info("Start data type conversion for LoadTsFileStatement: {}.", loadTsFileStatement);
- final List<Future<TSStatus>> executionFutures = new ArrayList<>();
// TODO: Use batch insert after Table model supports insertMultiTablets
for (final File file : loadTsFileStatement.getTsFiles()) {
try (final TsFileInsertionEventTableParser parser =
@@ -105,7 +100,10 @@ public class LoadTableStatementDataTypeConvertExecutionVisitor
.constructStatement(),
loadTsFileStatement.isConvertOnTypeMismatch());
- executionFutures.add(executeInsertTabletWithRetry(statement, databaseName));
+ if (!handleTSStatus(
+ executeInsertTabletWithRetry(statement, databaseName), loadTsFileStatement)) {
+ return Optional.empty();
+ }
}
} catch (final Exception e) {
LOGGER.warn(
@@ -114,25 +112,6 @@ public class LoadTableStatementDataTypeConvertExecutionVisitor
}
}
- for (final Future<TSStatus> future : executionFutures) {
- try {
- final TSStatus result = future.get();
- if (!(result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- || result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()
- || result.getCode()
- == TSStatusCode.LOAD_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) {
- LOGGER.warn(
- "Failed to convert data type for LoadTsFileStatement: {}, status code is {}.",
- loadTsFileStatement,
- result.getCode());
- return Optional.empty();
- }
- } catch (ExecutionException | InterruptedException e) {
- LOGGER.warn("Exception occurs when executing insertion during tablet conversion: ", e);
- return Optional.empty();
- }
- }
-
if (loadTsFileStatement.isDeleteAfterLoad()) {
loadTsFileStatement
.getTsFiles()
@@ -152,38 +131,33 @@ public class LoadTableStatementDataTypeConvertExecutionVisitor
return Optional.of(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
- private Future<TSStatus> executeInsertTabletWithRetry(
+ private TSStatus executeInsertTabletWithRetry(
final LoadConvertedInsertTabletStatement statement, final String databaseName) {
- return getExecutorPool()
- .submit(
- () -> {
- TSStatus result;
- try {
- result =
- statement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(statement, databaseName));
-
- // Retry max 5 times if the write process is rejected
- for (int i = 0;
- i < 5
- && result.getCode()
- == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
- i++) {
- Thread.sleep(100L * (i + 1));
- result =
- statement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(statement, databaseName));
- }
- } catch (final Exception e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- result =
- statement.accept(LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
- }
- return result;
- });
+ TSStatus result;
+ try {
+ result =
+ statement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(statement, databaseName));
+
+ // Retry max 5 times if the write process is rejected
+ for (int i = 0;
+ i < 5
+ && result.getCode()
+ == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
+ i++) {
+ Thread.sleep(100L * (i + 1));
+ result =
+ statement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(statement, databaseName));
+ }
+ } catch (final Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ result = statement.accept(LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
+ }
+ return result;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
index 838443a8b2e..bd20c652ddd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java
@@ -20,9 +20,6 @@
package org.apache.iotdb.db.storageengine.load.converter;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
-import org.apache.iotdb.commons.concurrent.ThreadName;
-import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReq;
@@ -49,14 +46,6 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil.calculateTabletSizeInBytes;
@@ -71,9 +60,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
.getConfig()
.getLoadTsFileTabletConversionBatchMemorySizeInBytes();
- private static final AtomicReference<WrappedThreadPoolExecutor> executorPool =
- new AtomicReference<>();
-
private final StatementExecutor statementExecutor;
@FunctionalInterface
@@ -81,21 +67,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
TSStatus execute(final Statement statement);
}
- public static class CallerBlocksPolicy implements RejectedExecutionHandler {
- public CallerBlocksPolicy() {}
-
- public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
- if (!e.isShutdown()) {
- try {
- e.getQueue().put(r);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new RejectedExecutionException("task " + r + " rejected from " + e, ie);
- }
- }
- }
- }
-
public LoadTreeStatementDataTypeConvertExecutionVisitor(
final StatementExecutor statementExecutor) {
this.statementExecutor = statementExecutor;
@@ -119,7 +90,6 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
final List<Long> tabletRawReqSizes = new ArrayList<>();
try {
- final List<Future<TSStatus>> executionFutures = new ArrayList<>();
for (final File file : loadTsFileStatement.getTsFiles()) {
try (final TsFileInsertionEventScanParser parser =
new TsFileInsertionEventScanParser(
@@ -136,16 +106,9 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
continue;
}
- final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
- batchStatement.setInsertTabletStatementList(
- tabletRawReqs.stream()
- .map(
- req ->
- new LoadConvertedInsertTabletStatement(
- req.constructStatement(),
- loadTsFileStatement.isConvertOnTypeMismatch()))
- .collect(Collectors.toList()));
- executionFutures.add(executeInsertMultiTabletsWithRetry(batchStatement));
+ final TSStatus result =
+ executeInsertMultiTabletsWithRetry(
+ tabletRawReqs, loadTsFileStatement.isConvertOnTypeMismatch());
for (final long memoryCost : tabletRawReqSizes) {
block.reduceMemoryUsage(memoryCost);
@@ -153,6 +116,10 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
tabletRawReqs.clear();
tabletRawReqSizes.clear();
+ if (!handleTSStatus(result, loadTsFileStatement)) {
+ return Optional.empty();
+ }
+
tabletRawReqs.add(tabletRawReq);
tabletRawReqSizes.add(curMemory);
block.addMemoryUsage(curMemory);
@@ -166,36 +133,22 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
if (!tabletRawReqs.isEmpty()) {
try {
- final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
- batchStatement.setInsertTabletStatementList(
- tabletRawReqs.stream()
- .map(
- req ->
- new LoadConvertedInsertTabletStatement(
- req.constructStatement(),
- loadTsFileStatement.isConvertOnTypeMismatch()))
- .collect(Collectors.toList()));
- executionFutures.add(executeInsertMultiTabletsWithRetry(batchStatement));
+ final TSStatus result =
+ executeInsertMultiTabletsWithRetry(
+ tabletRawReqs, loadTsFileStatement.isConvertOnTypeMismatch());
for (final long memoryCost : tabletRawReqSizes) {
block.reduceMemoryUsage(memoryCost);
}
tabletRawReqs.clear();
tabletRawReqSizes.clear();
- } catch (final Exception e) {
- LOGGER.warn(
- "Failed to convert data type for LoadTsFileStatement: {}.", loadTsFileStatement, e);
- return Optional.empty();
- }
- }
- for (final Future<TSStatus> future : executionFutures) {
- try {
- if (!handleTSStatus(future.get(), loadTsFileStatement)) {
+ if (!handleTSStatus(result, loadTsFileStatement)) {
return Optional.empty();
}
- } catch (ExecutionException | InterruptedException e) {
- LOGGER.warn("Exception occurs when executing insertion during tablet conversion: ", e);
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Failed to convert data type for LoadTsFileStatement: {}.", loadTsFileStatement, e);
return Optional.empty();
}
}
@@ -227,71 +180,46 @@ public class LoadTreeStatementDataTypeConvertExecutionVisitor
return Optional.of(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
- private Future<TSStatus> executeInsertMultiTabletsWithRetry(
- final InsertMultiTabletsStatement batchStatement) {
- return getExecutorPool()
- .submit(
- () -> {
- TSStatus result;
- try {
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(batchStatement));
-
- // Retry max 5 times if the write process is rejected
- for (int i = 0;
- i < 5
- && result.getCode()
- == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
- i++) {
- Thread.sleep(100L * (i + 1));
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
- statementExecutor.execute(batchStatement));
- }
- } catch (final Exception e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- result =
- batchStatement.accept(
- LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
- }
- return result;
- });
- }
-
- public static WrappedThreadPoolExecutor getExecutorPool() {
- if (executorPool.get() == null) {
- synchronized (executorPool) {
- if (executorPool.get() == null) {
- executorPool.set(
- new WrappedThreadPoolExecutor(
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount(),
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount(),
- 0L,
- TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(
- IoTDBDescriptor.getInstance()
- .getConfig()
- .getLoadTsFileTabletConversionThreadCount()),
- new IoTThreadFactory(ThreadName.LOAD_DATATYPE_CONVERT_POOL.getName()),
- ThreadName.LOAD_DATATYPE_CONVERT_POOL.getName(),
- new CallerBlocksPolicy()));
- }
+ private TSStatus executeInsertMultiTabletsWithRetry(
+ final List<PipeTransferTabletRawReq> tabletRawReqs, boolean isConvertOnTypeMismatch) {
+ final InsertMultiTabletsStatement batchStatement = new InsertMultiTabletsStatement();
+ batchStatement.setInsertTabletStatementList(
+ tabletRawReqs.stream()
+ .map(
+ req ->
+ new LoadConvertedInsertTabletStatement(
+ req.constructStatement(), isConvertOnTypeMismatch))
+ .collect(Collectors.toList()));
+
+ TSStatus result;
+ try {
+ result =
+ batchStatement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(batchStatement));
+
+ // Retry max 5 times if the write process is rejected
+ for (int i = 0;
+ i < 5
+ && result.getCode()
+ == TSStatusCode.LOAD_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode();
+ i++) {
+ Thread.sleep(100L * (i + 1));
+ result =
+ batchStatement.accept(
+ LoadTsFileDataTypeConverter.STATEMENT_STATUS_VISITOR,
+ statementExecutor.execute(batchStatement));
+ }
+ } catch (final Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
}
+ result = batchStatement.accept(LoadTsFileDataTypeConverter.STATEMENT_EXCEPTION_VISITOR, e);
}
- return executorPool.get();
+ return result;
}
- private static boolean handleTSStatus(
- final TSStatus result, final LoadTsFileStatement loadTsFileStatement) {
+ public static boolean handleTSStatus(final TSStatus result, final Object loadTsFileStatement) {
if (!(result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
|| result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()
|| result.getCode() == TSStatusCode.LOAD_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
index 57457708e40..fea85b11f8d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
@@ -26,8 +26,11 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.protocol.session.IClientSession;
import org.apache.iotdb.db.protocol.session.InternalClientSession;
import org.apache.iotdb.db.protocol.session.SessionManager;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
+import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager;
+import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
@@ -55,6 +58,7 @@ public class LoadTsFileDataTypeConverter {
STATEMENT_EXCEPTION_VISITOR = new LoadConvertedInsertTabletStatementExceptionVisitor();
private final boolean isGeneratedByPipe;
+ private final MPPQueryContext context;
private final SqlParser relationalSqlParser = new SqlParser();
private final LoadTableStatementDataTypeConvertExecutionVisitor
@@ -64,7 +68,9 @@ public class LoadTsFileDataTypeConverter {
treeStatementDataTypeConvertExecutionVisitor =
new LoadTreeStatementDataTypeConvertExecutionVisitor(this::executeForTreeModel);
- public LoadTsFileDataTypeConverter(final boolean isGeneratedByPipe) {
+ public LoadTsFileDataTypeConverter(
+ final MPPQueryContext context, final boolean isGeneratedByPipe) {
+ this.context = context;
this.isGeneratedByPipe = isGeneratedByPipe;
}
@@ -113,6 +119,7 @@ public class LoadTsFileDataTypeConverter {
}
public Optional<TSStatus> convertForTreeModel(final LoadTsFileStatement loadTsFileTreeStatement) {
+ DataNodeSchemaLockManager.getInstance().releaseReadLock(context);
try {
return loadTsFileTreeStatement.accept(treeStatementDataTypeConvertExecutionVisitor, null);
} catch (Exception e) {
@@ -120,36 +127,24 @@ public class LoadTsFileDataTypeConverter {
"Failed to convert data types for tree model statement {}.", loadTsFileTreeStatement, e);
return Optional.of(
new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage()));
+ } finally {
+ DataNodeSchemaLockManager.getInstance()
+ .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION_TREE);
}
}
private TSStatus executeForTreeModel(final Statement statement) {
- final IClientSession session =
- new InternalClientSession(
- String.format(
- "%s_%s",
- LoadTsFileDataTypeConverter.class.getSimpleName(),
- Thread.currentThread().getName()));
- session.setUsername(AuthorityChecker.SUPER_USER);
- session.setClientVersion(IoTDBConstant.ClientVersion.V_1_0);
- session.setZoneId(ZoneId.systemDefault());
-
- SESSION_MANAGER.registerSession(session);
- try {
- return Coordinator.getInstance()
- .executeForTreeModel(
- isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement,
- SESSION_MANAGER.requestQueryId(),
- SESSION_MANAGER.getSessionInfo(session),
- "",
- ClusterPartitionFetcher.getInstance(),
- ClusterSchemaFetcher.getInstance(),
- IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(),
- false)
- .status;
- } finally {
- SESSION_MANAGER.removeCurrSession();
- }
+ return Coordinator.getInstance()
+ .executeForTreeModel(
+ isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement,
+ SESSION_MANAGER.requestQueryId(),
+ SESSION_MANAGER.getSessionInfo(SESSION_MANAGER.getCurrSession()),
+ "",
+ ClusterPartitionFetcher.getInstance(),
+ ClusterSchemaFetcher.getInstance(),
+ IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(),
+ false)
+ .status;
}
public boolean isSuccessful(final TSStatus status) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java
index f473a56a447..5dcc5d8943c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/execution/SubscriptionSubtaskExecutor.java
@@ -26,25 +26,22 @@ import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.db.pipe.agent.task.execution.PipeConnectorSubtaskExecutor;
import org.apache.iotdb.db.subscription.task.subtask.SubscriptionReceiverSubtask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
public class SubscriptionSubtaskExecutor extends PipeConnectorSubtaskExecutor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionSubtaskExecutor.class);
+ private static final AtomicInteger id = new AtomicInteger(0);
private final AtomicLong submittedReceiverSubtasks = new AtomicLong(0);
public SubscriptionSubtaskExecutor() {
super(
SubscriptionConfig.getInstance().getSubscriptionSubtaskExecutorMaxThreadNum(),
- ThreadName.SUBSCRIPTION_EXECUTOR_POOL);
+ ThreadName.SUBSCRIPTION_EXECUTOR_POOL.getName() + "-" + id.getAndIncrement());
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java
index 003d712cc5b..7d26701004a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskConnectorStage.java
@@ -36,7 +36,7 @@ public class SubscriptionTaskConnectorStage extends PipeTaskConnectorStage {
PipeParameters pipeConnectorParameters,
int regionId,
PipeConnectorSubtaskExecutor executor) {
- super(pipeName, creationTime, pipeConnectorParameters, regionId, executor);
+ super(pipeName, creationTime, pipeConnectorParameters, regionId, () -> executor);
}
@Override
@@ -44,7 +44,7 @@ public class SubscriptionTaskConnectorStage extends PipeTaskConnectorStage {
this.connectorSubtaskId =
SubscriptionConnectorSubtaskManager.instance()
.register(
- executor,
+ executor.get(),
pipeConnectorParameters,
new PipeTaskConnectorRuntimeEnvironment(pipeName, creationTime, regionId));
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
index b1580e91341..e30c76795d8 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeDataNodeThriftRequestTest.java
@@ -78,7 +78,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getTimestampPrecision(), deserializeReq.getTimestampPrecision());
}
@@ -101,7 +100,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getInsertNode(), deserializeReq.getInsertNode());
@@ -155,7 +153,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
}
@Test
@@ -193,7 +190,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getPlanNode(), deserializeReq.getPlanNode());
}
@@ -231,7 +227,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
final Statement statement =
req.constructStatement(); // will call PipeTransferTabletRawReq.sortTablet() here
@@ -378,8 +373,6 @@ public class PipeDataNodeThriftRequestTest {
final PipeTransferTabletBatchReq deserializedReq =
PipeTransferTabletBatchReq.fromTPipeTransferReq(req);
- Assert.assertArrayEquals(
- new byte[] {'a', 'b'}, deserializedReq.getBinaryReqs().get(0).getBody());
Assert.assertEquals(node, deserializedReq.getInsertNodeReqs().get(0).getInsertNode());
Assert.assertEquals(t, deserializedReq.getTabletReqs().get(0).getTablet());
Assert.assertFalse(deserializedReq.getTabletReqs().get(0).getIsAligned());
@@ -485,7 +478,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileName(), deserializeReq.getFileName());
Assert.assertEquals(req.getStartWritingOffset(), deserializeReq.getStartWritingOffset());
@@ -504,7 +496,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileName(), deserializeReq.getFileName());
Assert.assertEquals(req.getStartWritingOffset(), deserializeReq.getStartWritingOffset());
@@ -523,7 +514,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileName(), deserializeReq.getFileName());
Assert.assertEquals(req.getStartWritingOffset(), deserializeReq.getStartWritingOffset());
@@ -541,7 +531,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileName(), deserializeReq.getFileName());
Assert.assertEquals(req.getFileLength(), deserializeReq.getFileLength());
@@ -576,7 +565,6 @@ public class PipeDataNodeThriftRequestTest {
Assert.assertEquals(req.getVersion(), deserializeReq.getVersion());
Assert.assertEquals(req.getType(), deserializeReq.getType());
- Assert.assertArrayEquals(req.getBody(), deserializeReq.getBody());
Assert.assertEquals(req.getFileNames(), deserializeReq.getFileNames());
Assert.assertEquals(req.getFileLengths(), deserializeReq.getFileLengths());
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java
index 1318b466b79..5a23efe8b21 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java
@@ -563,6 +563,7 @@ public class TsFileInsertionEventParserTest {
false,
"",
new TsFileResource(tsFile),
+ null,
true,
false,
false,
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/PipeRealtimeExtractTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/PipeRealtimeExtractTest.java
index 63d5661c5c7..02d6f622cd8 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/PipeRealtimeExtractTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/PipeRealtimeExtractTest.java
@@ -20,7 +20,9 @@
package org.apache.iotdb.db.pipe.extractor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration;
import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskExtractorRuntimeEnvironment;
@@ -34,13 +36,13 @@ import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInser
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.junit.After;
import org.junit.Assert;
@@ -64,8 +66,6 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
-import static org.mockito.Mockito.mock;
-
public class PipeRealtimeExtractTest {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeRealtimeExtractTest.class);
@@ -111,36 +111,37 @@ public class PipeRealtimeExtractTest {
public void testRealtimeExtractProcess() {
// set up realtime extractor
- try (PipeRealtimeDataRegionLogExtractor extractor0 = new PipeRealtimeDataRegionLogExtractor();
- PipeRealtimeDataRegionHybridExtractor extractor1 =
+ try (final PipeRealtimeDataRegionLogExtractor extractor0 =
+ new PipeRealtimeDataRegionLogExtractor();
+ final PipeRealtimeDataRegionHybridExtractor extractor1 =
new PipeRealtimeDataRegionHybridExtractor();
- PipeRealtimeDataRegionTsFileExtractor extractor2 =
+ final PipeRealtimeDataRegionTsFileExtractor extractor2 =
new PipeRealtimeDataRegionTsFileExtractor();
- PipeRealtimeDataRegionHybridExtractor extractor3 =
+ final PipeRealtimeDataRegionHybridExtractor extractor3 =
new PipeRealtimeDataRegionHybridExtractor()) {
- PipeParameters parameters0 =
+ final PipeParameters parameters0 =
new PipeParameters(
new HashMap<String, String>() {
... 3280 lines suppressed ...