You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2025/04/27 09:29:42 UTC
(iotdb) branch master updated: Pipe: Implement hot loading of Pipe configuration parameters (#15292)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 15ad13c9207 Pipe: Implement hot loading of Pipe configuration parameters (#15292)
15ad13c9207 is described below
commit 15ad13c92079f8d1fdc8f10cdcc86f6b5140c26b
Author: Zhenyu Luo <lu...@gmail.com>
AuthorDate: Sun Apr 27 17:29:36 2025 +0800
Pipe: Implement hot loading of Pipe configuration parameters (#15292)
Co-authored-by: Steve Yurong Su <ro...@apache.org>
---
.../iotdb/tool/tsfile/ImportTsFileRemotely.java | 6 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 14 +-
.../PipeRealtimePriorityBlockingQueue.java | 26 +-
.../IoTDBDataNodeCacheLeaderClientManager.java | 12 +-
.../async/IoTDBDataRegionAsyncConnector.java | 31 +-
.../scan/TsFileInsertionEventScanParser.java | 5 +-
...ileInsertionEventTableParserTabletIterator.java | 16 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 5 +-
.../realtime/assigner/PipeDataRegionAssigner.java | 5 +-
.../pipe/processor/twostage/combiner/Combiner.java | 5 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 10 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 54 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 445 +++++++++++++
.../iotdb/commons/conf/CommonDescriptor.java | 486 +-------------
.../task/connection/BlockingPendingQueue.java | 13 +-
.../agent/task/execution/PipeSubtaskScheduler.java | 15 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 5 +-
.../config/PipeDescriptor.java} | 726 +++++----------------
.../commons/pipe/receiver/IoTDBFileReceiver.java | 18 +-
19 files changed, 745 insertions(+), 1152 deletions(-)
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileRemotely.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileRemotely.java
index 9f4e09fa364..97b335afe2a 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileRemotely.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileRemotely.java
@@ -53,7 +53,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.LockSupport;
public class ImportTsFileRemotely extends ImportTsFileBase {
@@ -64,9 +63,6 @@ public class ImportTsFileRemotely extends ImportTsFileBase {
private static final String LOAD_STRATEGY = "sync";
private static final Integer MAX_RETRY_COUNT = 3;
- private static final AtomicInteger CONNECTION_TIMEOUT_MS =
- new AtomicInteger(PipeConfig.getInstance().getPipeConnectorTransferTimeoutMs());
-
private IoTDBSyncClient client;
private static String host;
@@ -166,7 +162,7 @@ public class ImportTsFileRemotely extends ImportTsFileBase {
"Handshake error with target server ip: %s, port: %s, because: %s.",
client.getIpAddress(), client.getPort(), resp.getStatus()));
} else {
- client.setTimeout(CONNECTION_TIMEOUT_MS.get());
+ client.setTimeout(PipeConfig.getInstance().getPipeConnectorTransferTimeoutMs());
IOT_PRINTER.println(
String.format(
"Handshake success. Target server ip: %s, port: %s",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 2ac818de64c..d9073afe1f0 100755
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.conf.TrimProperties;
import org.apache.iotdb.commons.exception.BadNodeUrlException;
import org.apache.iotdb.commons.memory.MemoryManager;
+import org.apache.iotdb.commons.pipe.config.PipeDescriptor;
import org.apache.iotdb.commons.schema.SchemaConstant;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.NodeUrlUtils;
@@ -1994,14 +1995,7 @@ public class IoTDBDescriptor {
loadLoadTsFileHotModifiedProp(properties);
// update pipe config
- commonDescriptor
- .getConfig()
- .setPipeAllSinksRateLimitBytesPerSecond(
- Double.parseDouble(
- properties.getProperty(
- "pipe_all_sinks_rate_limit_bytes_per_second",
- ConfigurationFileUtils.getConfigurationDefaultValue(
- "pipe_all_sinks_rate_limit_bytes_per_second"))));
+ loadPipeHotModifiedProp(properties);
// update merge_threshold_of_explain_analyze
conf.setMergeThresholdOfExplainAnalyze(
@@ -2312,6 +2306,10 @@ public class IoTDBDescriptor {
ConfigurationFileUtils.getConfigurationDefaultValue("load_active_listening_fail_dir")));
}
+ private void loadPipeHotModifiedProp(TrimProperties properties) throws IOException {
+ PipeDescriptor.loadPipeProps(commonDescriptor.getConfig(), properties, true);
+ }
+
@SuppressWarnings("squid:S3518") // "proportionSum" can't be zero
private void loadUDFProps(TrimProperties properties) {
String initialByteArrayLengthForMemoryControl =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
index 61391d039af..c8b050a5cfe 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
@@ -38,15 +38,13 @@ import java.util.function.Consumer;
public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQueue<Event> {
+ private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
+
private final BlockingDeque<TsFileInsertionEvent> tsfileInsertEventDeque =
new LinkedBlockingDeque<>();
- private static final int POLL_TSFILE_THRESHOLD =
- PipeConfig.getInstance().getPipeRealTimeQueuePollTsFileThreshold();
private final AtomicInteger pollTsFileCounter = new AtomicInteger(0);
- private static final int POLL_HISTORICAL_TSFILE_THRESHOLD =
- Math.max(PipeConfig.getInstance().getPipeRealTimeQueuePollHistoricalTsFileThreshold(), 1);
private final AtomicLong pollHistoricalTsFileCounter = new AtomicLong(0);
public PipeRealtimePriorityBlockingQueue() {
@@ -85,9 +83,12 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
@Override
public Event directPoll() {
Event event = null;
- if (pollTsFileCounter.get() >= POLL_TSFILE_THRESHOLD) {
+ final int pollHistoricalTsFileThreshold =
+ PIPE_CONFIG.getPipeRealTimeQueuePollHistoricalTsFileThreshold();
+
+ if (pollTsFileCounter.get() >= PIPE_CONFIG.getPipeRealTimeQueuePollTsFileThreshold()) {
event =
- pollHistoricalTsFileCounter.incrementAndGet() % POLL_HISTORICAL_TSFILE_THRESHOLD == 0
+ pollHistoricalTsFileCounter.incrementAndGet() % pollHistoricalTsFileThreshold == 0
? tsfileInsertEventDeque.pollFirst()
: tsfileInsertEventDeque.pollLast();
pollTsFileCounter.set(0);
@@ -97,7 +98,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
event = super.directPoll();
if (Objects.isNull(event)) {
event =
- pollHistoricalTsFileCounter.incrementAndGet() % POLL_HISTORICAL_TSFILE_THRESHOLD == 0
+ pollHistoricalTsFileCounter.incrementAndGet() % pollHistoricalTsFileThreshold == 0
? tsfileInsertEventDeque.pollFirst()
: tsfileInsertEventDeque.pollLast();
}
@@ -123,9 +124,12 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
@Override
public Event waitedPoll() {
Event event = null;
- if (pollTsFileCounter.get() >= POLL_TSFILE_THRESHOLD) {
+ final int pollHistoricalTsFileThreshold =
+ PIPE_CONFIG.getPipeRealTimeQueuePollHistoricalTsFileThreshold();
+
+ if (pollTsFileCounter.get() >= PIPE_CONFIG.getPipeRealTimeQueuePollTsFileThreshold()) {
event =
- pollHistoricalTsFileCounter.incrementAndGet() % POLL_HISTORICAL_TSFILE_THRESHOLD == 0
+ pollHistoricalTsFileCounter.incrementAndGet() % pollHistoricalTsFileThreshold == 0
? tsfileInsertEventDeque.pollFirst()
: tsfileInsertEventDeque.pollLast();
pollTsFileCounter.set(0);
@@ -135,7 +139,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
event = super.directPoll();
if (event == null && !tsfileInsertEventDeque.isEmpty()) {
event =
- pollHistoricalTsFileCounter.incrementAndGet() % POLL_HISTORICAL_TSFILE_THRESHOLD == 0
+ pollHistoricalTsFileCounter.incrementAndGet() % pollHistoricalTsFileThreshold == 0
? tsfileInsertEventDeque.pollFirst()
: tsfileInsertEventDeque.pollLast();
}
@@ -149,7 +153,7 @@ public class PipeRealtimePriorityBlockingQueue extends UnboundedBlockingPendingQ
event = super.waitedPoll();
if (Objects.isNull(event)) {
event =
- pollHistoricalTsFileCounter.incrementAndGet() % POLL_HISTORICAL_TSFILE_THRESHOLD == 0
+ pollHistoricalTsFileCounter.incrementAndGet() % pollHistoricalTsFileThreshold == 0
? tsfileInsertEventDeque.pollFirst()
: tsfileInsertEventDeque.pollLast();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java
index 8b88a1c36ee..799cd3bc8c4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeCacheLeaderClientManager.java
@@ -52,10 +52,6 @@ public interface IoTDBDataNodeCacheLeaderClientManager {
public LeaderCacheManager() {
final long initMemorySizeInBytes =
PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes() / 10;
- final long maxMemorySizeInBytes =
- (long)
- (PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- * CONFIG.getPipeLeaderCacheMemoryUsagePercentage());
// properties required by pipe memory control framework
final PipeMemoryBlock allocatedMemoryBlock =
@@ -72,7 +68,13 @@ public interface IoTDBDataNodeCacheLeaderClientManager {
newMemory);
})
.setExpandMethod(
- oldMemory -> Math.min(Math.max(oldMemory, 1) * 2, maxMemorySizeInBytes))
+ oldMemory ->
+ Math.min(
+ Math.max(oldMemory, 1) * 2,
+ (long)
+ (PipeDataNodeResourceManager.memory()
+ .getTotalNonFloatingMemorySizeInBytes()
+ * CONFIG.getPipeLeaderCacheMemoryUsagePercentage())))
.setExpandCallback(
(oldMemory, newMemory) -> {
memoryUsageCheatFactor.updateAndGet(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
index b8ea083a113..35e09c4fba7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java
@@ -101,14 +101,6 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
private final BlockingQueue<Event> retryEventQueue = new LinkedBlockingQueue<>();
private final PipeDataRegionEventCounter retryEventQueueEventCounter =
new PipeDataRegionEventCounter();
- private final int forcedRetryTsFileEventQueueSizeThreshold =
- PipeConfig.getInstance().getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold();
- private final int forcedRetryTabletEventQueueSizeThreshold =
- PipeConfig.getInstance().getPipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold();
- private final int forcedRetryTotalEventQueueSizeThreshold =
- PipeConfig.getInstance().getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold();
- private final long maxRetryExecutionTimeMsPerCall =
- PipeConfig.getInstance().getPipeAsyncConnectorMaxRetryExecutionTimeMsPerCall();
private IoTDBDataNodeAsyncClientManager clientManager;
@@ -468,10 +460,14 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
if (retryEventQueue.isEmpty()
|| (!forced
&& retryEventQueueEventCounter.getTabletInsertionEventCount()
- < forcedRetryTabletEventQueueSizeThreshold
+ < PipeConfig.getInstance()
+ .getPipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold()
&& retryEventQueueEventCounter.getTsFileInsertionEventCount()
- < forcedRetryTsFileEventQueueSizeThreshold
- && retryEventQueue.size() < forcedRetryTotalEventQueueSizeThreshold)) {
+ < PipeConfig.getInstance()
+ .getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold()
+ && retryEventQueue.size()
+ < PipeConfig.getInstance()
+ .getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold())) {
return;
}
@@ -515,12 +511,17 @@ public class IoTDBDataRegionAsyncConnector extends IoTDBConnector {
}
// Stop retrying if the execution time exceeds the threshold for better realtime performance
- if (System.currentTimeMillis() - retryStartTime > maxRetryExecutionTimeMsPerCall) {
+ if (System.currentTimeMillis() - retryStartTime
+ > PipeConfig.getInstance().getPipeAsyncConnectorMaxRetryExecutionTimeMsPerCall()) {
if (retryEventQueueEventCounter.getTabletInsertionEventCount()
- < forcedRetryTabletEventQueueSizeThreshold
+ < PipeConfig.getInstance()
+ .getPipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold()
&& retryEventQueueEventCounter.getTsFileInsertionEventCount()
- < forcedRetryTsFileEventQueueSizeThreshold
- && retryEventQueue.size() < forcedRetryTotalEventQueueSizeThreshold) {
+ < PipeConfig.getInstance()
+ .getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold()
+ && retryEventQueue.size()
+ < PipeConfig.getInstance()
+ .getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold()) {
return;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java
index 83dbdf39d05..9dcbc01bd72 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java
@@ -64,8 +64,9 @@ import java.util.Objects;
public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser {
- private static final int PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH =
+ private final int pipeMaxAlignedSeriesNumInOneBatch =
PipeConfig.getInstance().getPipeMaxAlignedSeriesNumInOneBatch();
+
private final long startTime;
private final long endTime;
private final Filter filter;
@@ -463,7 +464,7 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser {
boolean needReturn = false;
if (lastIndex >= 0
&& (valueIndex != lastIndex
- || valueChunkList.size() >= PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH)) {
+ || valueChunkList.size() >= pipeMaxAlignedSeriesNumInOneBatch)) {
needReturn = recordAlignedChunk(valueChunkList, marker);
}
lastIndex = valueIndex;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java
index c080e1dc152..661c5cc6feb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java
@@ -57,8 +57,10 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
public class TsFileInsertionEventTableParserTabletIterator implements Iterator<Tablet> {
- private static final int PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH =
+
+ private final int pipeMaxAlignedSeriesNumInOneBatch =
PipeConfig.getInstance().getPipeMaxAlignedSeriesNumInOneBatch();
+
private final long startTime;
private final long endTime;
@@ -223,9 +225,9 @@ public class TsFileInsertionEventTableParserTabletIterator implements Iterator<T
deviceMetaIterator = metadataQuerier.deviceIterator(tableRoot, null);
final int columnSchemaSize = tableSchema.getColumnSchemas().size();
- dataTypeList = new ArrayList<>(PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH);
- columnTypes = new ArrayList<>(PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH);
- measurementList = new ArrayList<>(PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH);
+ dataTypeList = new ArrayList<>(pipeMaxAlignedSeriesNumInOneBatch);
+ columnTypes = new ArrayList<>(pipeMaxAlignedSeriesNumInOneBatch);
+ measurementList = new ArrayList<>(pipeMaxAlignedSeriesNumInOneBatch);
for (int i = 0; i < columnSchemaSize; i++) {
final IMeasurementSchema schema = tableSchema.getColumnSchemas().get(i);
@@ -328,7 +330,7 @@ public class TsFileInsertionEventTableParserTabletIterator implements Iterator<T
timeChunk.getData().rewind();
long size = timeChunkSize;
- final List<Chunk> valueChunkList = new ArrayList<>(PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH);
+ final List<Chunk> valueChunkList = new ArrayList<>(pipeMaxAlignedSeriesNumInOneBatch);
// To ensure that the Tablet has the same alignedChunk column as the current one,
// you need to create a new Tablet to fill in the data.
@@ -336,7 +338,7 @@ public class TsFileInsertionEventTableParserTabletIterator implements Iterator<T
// Need to ensure that columnTypes recreates an array
final List<Tablet.ColumnCategory> categories =
- new ArrayList<>(deviceIdSize + PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH);
+ new ArrayList<>(deviceIdSize + pipeMaxAlignedSeriesNumInOneBatch);
for (int i = 0; i < deviceIdSize; i++) {
categories.add(Tablet.ColumnCategory.TAG);
}
@@ -361,7 +363,7 @@ public class TsFileInsertionEventTableParserTabletIterator implements Iterator<T
valueChunkList.add(chunk);
}
- if (offset - startOffset >= PIPE_MAX_ALIGNED_SERIES_NUM_IN_ONE_BATCH) {
+ if (offset - startOffset >= pipeMaxAlignedSeriesNumInOneBatch) {
break;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
index 8a3826b66a8..1a66fa458da 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionHybridExtractor.java
@@ -48,9 +48,6 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
private static final Logger LOGGER =
LoggerFactory.getLogger(PipeRealtimeDataRegionHybridExtractor.class);
- private final boolean isPipeEpochKeepTsFileAfterStuckRestartEnabled =
- PipeConfig.getInstance().isPipeEpochKeepTsFileAfterStuckRestartEnabled();
-
@Override
protected void doExtract(final PipeRealtimeEvent event) {
final Event eventToExtract = event.getEvent();
@@ -230,7 +227,7 @@ public class PipeRealtimeDataRegionHybridExtractor extends PipeRealtimeDataRegio
}
private boolean isPipeTaskCurrentlyRestarted(final PipeRealtimeEvent event) {
- if (!isPipeEpochKeepTsFileAfterStuckRestartEnabled) {
+ if (!PipeConfig.getInstance().isPipeEpochKeepTsFileAfterStuckRestartEnabled()) {
return false;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
index 3184d6a3524..ce679516dcf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/assigner/PipeDataRegionAssigner.java
@@ -54,8 +54,7 @@ public class PipeDataRegionAssigner implements Closeable {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeDataRegionAssigner.class);
- private static final int nonForwardingEventsProgressReportInterval =
- PipeConfig.getInstance().getPipeNonForwardingEventsProgressReportInterval();
+ private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
/**
* The {@link PipeDataRegionMatcher} is used to match the event with the extractor based on the
@@ -150,7 +149,7 @@ public class PipeDataRegionAssigner implements Closeable {
// The frequency of progress reports is limited by the counter, while progress
// reports to TsFileInsertionEvent are not limited.
if (!(event.getEvent() instanceof TsFileInsertionEvent)) {
- if (counter < nonForwardingEventsProgressReportInterval) {
+ if (counter < PIPE_CONFIG.getPipeNonForwardingEventsProgressReportInterval()) {
counter++;
return;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java
index 136e760e44f..2ad490b1d03 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/combiner/Combiner.java
@@ -38,8 +38,6 @@ public class Combiner {
private static final Logger LOGGER = LoggerFactory.getLogger(Combiner.class);
- private static final long MAX_COMBINER_LIVE_TIME_IN_MS =
- PipeConfig.getInstance().getTwoStageAggregateMaxCombinerLiveTimeInMs();
private final long creationTimeInMs;
private final Operator operator;
@@ -98,7 +96,8 @@ public class Combiner {
}
public boolean isOutdated() {
- return System.currentTimeMillis() - creationTimeInMs > MAX_COMBINER_LIVE_TIME_IN_MS;
+ return System.currentTimeMillis() - creationTimeInMs
+ > PipeConfig.getInstance().getTwoStageAggregateMaxCombinerLiveTimeInMs();
}
public boolean isComplete() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index 513c7b0790f..5ba383eb885 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -178,8 +178,8 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
private static final SessionManager SESSION_MANAGER = SessionManager.getInstance();
- private static final double ACTUAL_TO_ESTIMATED_MEMORY_RATIO =
- PipeConfig.getInstance().getPipeReceiverActualToEstimatedMemoryRatio();
+ private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
+
private PipeMemoryBlock allocatedMemoryBlock;
static {
@@ -772,12 +772,14 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
private TSStatus executeStatementAndClassifyExceptions(final Statement statement) {
long estimatedMemory = 0L;
+ final double pipeReceiverActualToEstimatedMemoryRatio =
+ PIPE_CONFIG.getPipeReceiverActualToEstimatedMemoryRatio();
try {
if (statement instanceof InsertBaseStatement) {
estimatedMemory = ((InsertBaseStatement) statement).ramBytesUsed();
allocatedMemoryBlock =
PipeDataNodeResourceManager.memory()
- .forceAllocate((long) (estimatedMemory * ACTUAL_TO_ESTIMATED_MEMORY_RATIO));
+ .forceAllocate((long) (estimatedMemory * pipeReceiverActualToEstimatedMemoryRatio));
}
final TSStatus result =
@@ -798,7 +800,7 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
String.format(
"Temporarily out of memory when executing statement %s, Requested memory: %s, used memory: %s, total memory: %s",
statement,
- estimatedMemory * ACTUAL_TO_ESTIMATED_MEMORY_RATIO,
+ estimatedMemory * pipeReceiverActualToEstimatedMemoryRatio,
PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes(),
PipeDataNodeResourceManager.memory().getFreeMemorySizeInBytes());
if (LOGGER.isDebugEnabled()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
index 0c7a473950d..73252aa3797 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
@@ -40,17 +40,11 @@ public class PipeMemoryManager {
private static final Logger LOGGER = LoggerFactory.getLogger(PipeMemoryManager.class);
+ private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
+
private static final boolean PIPE_MEMORY_MANAGEMENT_ENABLED =
PipeConfig.getInstance().getPipeMemoryManagementEnabled();
- private static final int MEMORY_ALLOCATE_MAX_RETRIES =
- PipeConfig.getInstance().getPipeMemoryAllocateMaxRetries();
- private static final long MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS =
- PipeConfig.getInstance().getPipeMemoryAllocateRetryIntervalInMs();
-
- private static final long MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES =
- PipeConfig.getInstance().getPipeMemoryAllocateMinSizeInBytes();
-
// TODO @spricoder: consider combine memory block and used MemorySizeInBytes
private IMemoryBlock memoryBlock =
IoTDBDescriptor.getInstance()
@@ -60,15 +54,8 @@ public class PipeMemoryManager {
private static final double EXCEED_PROTECT_THRESHOLD = 0.95;
- // To avoid too much parsed events causing OOM. If total tablet memory size exceeds this
- // threshold, allocations of memory block for tablets will be rejected.
- private static final double TABLET_MEMORY_REJECT_THRESHOLD =
- PipeConfig.getInstance().getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold();
private volatile long usedMemorySizeInBytesOfTablets;
- // Used to control the memory allocated for managing slice tsfile.
- private static final double TS_FILE_MEMORY_REJECT_THRESHOLD =
- PipeConfig.getInstance().getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold();
private volatile long usedMemorySizeInBytesOfTsFiles;
private static final double FLOATING_MEMORY_RATIO =
@@ -96,17 +83,20 @@ public class PipeMemoryManager {
// block does not exceed TABLET_MEMORY_REJECT_THRESHOLD + TS_FILE_MEMORY_REJECT_THRESHOLD
private double allowedMaxMemorySizeInBytesOfTabletsAndTsFiles() {
- return (TABLET_MEMORY_REJECT_THRESHOLD + TS_FILE_MEMORY_REJECT_THRESHOLD)
+ return (PIPE_CONFIG.getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold()
+ + PIPE_CONFIG.getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold())
* getTotalNonFloatingMemorySizeInBytes();
}
private double allowedMaxMemorySizeInBytesOfTablets() {
- return (TABLET_MEMORY_REJECT_THRESHOLD + TS_FILE_MEMORY_REJECT_THRESHOLD / 2)
+ return (PIPE_CONFIG.getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold()
+ + PIPE_CONFIG.getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold() / 2)
* getTotalNonFloatingMemorySizeInBytes();
}
private double allowedMaxMemorySizeInBytesOfTsTiles() {
- return (TS_FILE_MEMORY_REJECT_THRESHOLD + TABLET_MEMORY_REJECT_THRESHOLD / 2)
+ return (PIPE_CONFIG.getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold()
+ + PIPE_CONFIG.getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold() / 2)
* getTotalNonFloatingMemorySizeInBytes();
}
@@ -161,13 +151,13 @@ public class PipeMemoryManager {
return (PipeTabletMemoryBlock) registerMemoryBlock(0, PipeMemoryBlockType.TABLET);
}
- for (int i = 1; i <= MEMORY_ALLOCATE_MAX_RETRIES; i++) {
+ for (int i = 1, size = PIPE_CONFIG.getPipeMemoryAllocateMaxRetries(); i <= size; i++) {
if (isHardEnough4TabletParsing()) {
break;
}
try {
- Thread.sleep(MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS);
+ Thread.sleep(PIPE_CONFIG.getPipeMemoryAllocateRetryIntervalInMs());
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
LOGGER.warn("forceAllocateWithRetry: interrupted while waiting for available memory", ex);
@@ -203,13 +193,13 @@ public class PipeMemoryManager {
return (PipeTsFileMemoryBlock) registerMemoryBlock(0, PipeMemoryBlockType.TS_FILE);
}
- for (int i = 1; i <= MEMORY_ALLOCATE_MAX_RETRIES; i++) {
+ for (int i = 1, size = PIPE_CONFIG.getPipeMemoryAllocateMaxRetries(); i <= size; i++) {
if (isHardEnough4TsFileSlicing()) {
break;
}
try {
- Thread.sleep(MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS);
+ Thread.sleep(PIPE_CONFIG.getPipeMemoryAllocateRetryIntervalInMs());
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
LOGGER.warn("forceAllocateWithRetry: interrupted while waiting for available memory", ex);
@@ -248,7 +238,8 @@ public class PipeMemoryManager {
}
}
- for (int i = 1; i <= MEMORY_ALLOCATE_MAX_RETRIES; i++) {
+ final int memoryAllocateMaxRetries = PIPE_CONFIG.getPipeMemoryAllocateMaxRetries();
+ for (int i = 1; i <= memoryAllocateMaxRetries; i++) {
if (getTotalNonFloatingMemorySizeInBytes() - memoryBlock.getUsedMemoryInBytes()
>= sizeInBytes) {
return registerMemoryBlock(sizeInBytes, type);
@@ -256,7 +247,7 @@ public class PipeMemoryManager {
try {
tryShrinkUntilFreeMemorySatisfy(sizeInBytes);
- this.wait(MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS);
+ this.wait(PIPE_CONFIG.getPipeMemoryAllocateRetryIntervalInMs());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOGGER.warn("forceAllocate: interrupted while waiting for available memory", e);
@@ -268,7 +259,7 @@ public class PipeMemoryManager {
"forceAllocate: failed to allocate memory after %d retries, "
+ "total memory size %d bytes, used memory size %d bytes, "
+ "requested memory size %d bytes",
- MEMORY_ALLOCATE_MAX_RETRIES,
+ memoryAllocateMaxRetries,
getTotalNonFloatingMemorySizeInBytes(),
memoryBlock.getUsedMemoryInBytes(),
sizeInBytes));
@@ -311,7 +302,8 @@ public class PipeMemoryManager {
}
long sizeInBytes = targetSize - oldSize;
- for (int i = 1; i <= MEMORY_ALLOCATE_MAX_RETRIES; i++) {
+ final int memoryAllocateMaxRetries = PIPE_CONFIG.getPipeMemoryAllocateMaxRetries();
+ for (int i = 1; i <= memoryAllocateMaxRetries; i++) {
if (getTotalNonFloatingMemorySizeInBytes() - memoryBlock.getUsedMemoryInBytes()
>= sizeInBytes) {
memoryBlock.forceAllocateWithoutLimitation(sizeInBytes);
@@ -327,7 +319,7 @@ public class PipeMemoryManager {
try {
tryShrinkUntilFreeMemorySatisfy(sizeInBytes);
- this.wait(MEMORY_ALLOCATE_RETRY_INTERVAL_IN_MS);
+ this.wait(PIPE_CONFIG.getPipeMemoryAllocateRetryIntervalInMs());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOGGER.warn("forceResize: interrupted while waiting for available memory", e);
@@ -339,7 +331,7 @@ public class PipeMemoryManager {
"forceResize: failed to allocate memory after %d retries, "
+ "total memory size %d bytes, used memory size %d bytes, "
+ "requested memory size %d bytes",
- MEMORY_ALLOCATE_MAX_RETRIES,
+ memoryAllocateMaxRetries,
getTotalNonFloatingMemorySizeInBytes(),
memoryBlock.getUsedMemoryInBytes(),
sizeInBytes));
@@ -393,7 +385,9 @@ public class PipeMemoryManager {
}
long sizeToAllocateInBytes = sizeInBytes;
- while (sizeToAllocateInBytes > MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES) {
+ final long memoryAllocateMinSizeInBytes = PIPE_CONFIG.getPipeMemoryAllocateMinSizeInBytes();
+
+ while (sizeToAllocateInBytes > memoryAllocateMinSizeInBytes) {
if (getTotalNonFloatingMemorySizeInBytes() - memoryBlock.getUsedMemoryInBytes()
>= sizeToAllocateInBytes) {
LOGGER.info(
@@ -411,7 +405,7 @@ public class PipeMemoryManager {
sizeToAllocateInBytes =
Math.max(
customAllocateStrategy.applyAsLong(sizeToAllocateInBytes),
- MEMORY_ALLOCATE_MIN_SIZE_IN_BYTES);
+ memoryAllocateMinSizeInBytes);
}
if (tryShrinkUntilFreeMemorySatisfy(sizeToAllocateInBytes)) {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index ef289d75c19..894a28487fa 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@@ -705,8 +706,15 @@ public class CommonConfig {
public void setPipeNonForwardingEventsProgressReportInterval(
int pipeNonForwardingEventsProgressReportInterval) {
+ if (this.pipeNonForwardingEventsProgressReportInterval
+ == pipeNonForwardingEventsProgressReportInterval) {
+ return;
+ }
this.pipeNonForwardingEventsProgressReportInterval =
pipeNonForwardingEventsProgressReportInterval;
+ logger.info(
+ "pipeNonForwardingEventsProgressReportInterval is set to {}.",
+ pipeNonForwardingEventsProgressReportInterval);
}
public String getPipeHardlinkBaseDirName() {
@@ -714,7 +722,11 @@ public class CommonConfig {
}
public void setPipeHardlinkBaseDirName(String pipeHardlinkBaseDirName) {
+ if (Objects.equals(pipeHardlinkBaseDirName, this.pipeHardlinkBaseDirName)) {
+ return;
+ }
this.pipeHardlinkBaseDirName = pipeHardlinkBaseDirName;
+ logger.info("pipeHardlinkBaseDirName is set to {}.", pipeHardlinkBaseDirName);
}
public String getPipeHardlinkTsFileDirName() {
@@ -722,7 +734,11 @@ public class CommonConfig {
}
public void setPipeHardlinkTsFileDirName(String pipeTsFileDirName) {
+ if (Objects.equals(this.pipeHardlinkTsFileDirName, pipeTsFileDirName)) {
+ return;
+ }
this.pipeHardlinkTsFileDirName = pipeTsFileDirName;
+ logger.info("pipeHardlinkTsFileDirName is set to {}.", pipeTsFileDirName);
}
public String getPipeHardlinkWALDirName() {
@@ -730,7 +746,11 @@ public class CommonConfig {
}
public void setPipeHardlinkWALDirName(String pipeWALDirName) {
+ if (Objects.equals(pipeWALDirName, this.pipeHardlinkWALDirName)) {
+ return;
+ }
this.pipeHardlinkWALDirName = pipeWALDirName;
+ logger.info("pipeHardlinkWALDirName is set to {}.", pipeWALDirName);
}
public boolean getPipeHardLinkWALEnabled() {
@@ -738,7 +758,11 @@ public class CommonConfig {
}
public void setPipeHardLinkWALEnabled(boolean pipeHardLinkWALEnabled) {
+ if (this.pipeHardLinkWALEnabled == pipeHardLinkWALEnabled) {
+ return;
+ }
this.pipeHardLinkWALEnabled = pipeHardLinkWALEnabled;
+ logger.info("pipeHardLinkWALEnabled is set to {}.", pipeHardLinkWALEnabled);
}
public boolean getPipeFileReceiverFsyncEnabled() {
@@ -746,7 +770,11 @@ public class CommonConfig {
}
public void setPipeFileReceiverFsyncEnabled(boolean pipeFileReceiverFsyncEnabled) {
+ if (this.pipeFileReceiverFsyncEnabled == pipeFileReceiverFsyncEnabled) {
+ return;
+ }
this.pipeFileReceiverFsyncEnabled = pipeFileReceiverFsyncEnabled;
+ logger.info("pipeFileReceiverFsyncEnabled is set to {}.", pipeFileReceiverFsyncEnabled);
}
public int getPipeDataStructureTabletRowSize() {
@@ -754,7 +782,11 @@ public class CommonConfig {
}
public void setPipeDataStructureTabletRowSize(int pipeDataStructureTabletRowSize) {
+ if (this.pipeDataStructureTabletRowSize == pipeDataStructureTabletRowSize) {
+ return;
+ }
this.pipeDataStructureTabletRowSize = pipeDataStructureTabletRowSize;
+ logger.info("pipeDataStructureTabletRowSize is set to {}.", pipeDataStructureTabletRowSize);
}
public int getPipeDataStructureTabletSizeInBytes() {
@@ -762,7 +794,12 @@ public class CommonConfig {
}
public void setPipeDataStructureTabletSizeInBytes(int pipeDataStructureTabletSizeInBytes) {
+ if (this.pipeDataStructureTabletSizeInBytes == pipeDataStructureTabletSizeInBytes) {
+ return;
+ }
this.pipeDataStructureTabletSizeInBytes = pipeDataStructureTabletSizeInBytes;
+ logger.info(
+ "pipeDataStructureTabletSizeInBytes is set to {}.", pipeDataStructureTabletSizeInBytes);
}
public double getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold() {
@@ -771,8 +808,15 @@ public class CommonConfig {
public void setPipeDataStructureTabletMemoryBlockAllocationRejectThreshold(
double pipeDataStructureTabletMemoryBlockAllocationRejectThreshold) {
+ if (this.pipeDataStructureTabletMemoryBlockAllocationRejectThreshold
+ == pipeDataStructureTabletMemoryBlockAllocationRejectThreshold) {
+ return;
+ }
this.pipeDataStructureTabletMemoryBlockAllocationRejectThreshold =
pipeDataStructureTabletMemoryBlockAllocationRejectThreshold;
+ logger.info(
+ "pipeDataStructureTabletMemoryBlockAllocationRejectThreshold is set to {}.",
+ pipeDataStructureTabletMemoryBlockAllocationRejectThreshold);
}
public double getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold() {
@@ -781,8 +825,15 @@ public class CommonConfig {
public void setPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold(
double pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold) {
+ if (this.pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold
+ == pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold) {
+ return;
+ }
this.pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold =
pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold;
+ logger.info(
+ "pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold is set to {}.",
+ pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold);
}
public double getPipeTotalFloatingMemoryProportion() {
@@ -790,7 +841,12 @@ public class CommonConfig {
}
public void setPipeTotalFloatingMemoryProportion(double pipeTotalFloatingMemoryProportion) {
+ if (this.pipeTotalFloatingMemoryProportion == pipeTotalFloatingMemoryProportion) {
+ return;
+ }
this.pipeTotalFloatingMemoryProportion = pipeTotalFloatingMemoryProportion;
+ logger.info(
+ "pipeTotalFloatingMemoryProportion is set to {}.", pipeTotalFloatingMemoryProportion);
}
public int getPipeExtractorAssignerDisruptorRingBufferSize() {
@@ -799,8 +855,15 @@ public class CommonConfig {
public void setPipeExtractorAssignerDisruptorRingBufferSize(
int pipeExtractorAssignerDisruptorRingBufferSize) {
+ if (this.pipeExtractorAssignerDisruptorRingBufferSize
+ == pipeExtractorAssignerDisruptorRingBufferSize) {
+ return;
+ }
this.pipeExtractorAssignerDisruptorRingBufferSize =
pipeExtractorAssignerDisruptorRingBufferSize;
+ logger.info(
+ "pipeExtractorAssignerDisruptorRingBufferSize is set to {}.",
+ pipeExtractorAssignerDisruptorRingBufferSize);
}
public long getPipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes() {
@@ -809,8 +872,15 @@ public class CommonConfig {
public void setPipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes(
long pipeExtractorAssignerDisruptorRingBufferEntrySize) {
+ if (pipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes
+ == pipeExtractorAssignerDisruptorRingBufferEntrySize) {
+ return;
+ }
this.pipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes =
pipeExtractorAssignerDisruptorRingBufferEntrySize;
+ logger.info(
+ "pipeExtractorAssignerDisruptorRingBufferEntrySize is set to {}.",
+ pipeExtractorAssignerDisruptorRingBufferEntrySize);
}
public int getPipeExtractorMatcherCacheSize() {
@@ -818,7 +888,11 @@ public class CommonConfig {
}
public void setPipeExtractorMatcherCacheSize(int pipeExtractorMatcherCacheSize) {
+ if (this.pipeExtractorMatcherCacheSize == pipeExtractorMatcherCacheSize) {
+ return;
+ }
this.pipeExtractorMatcherCacheSize = pipeExtractorMatcherCacheSize;
+ logger.info("pipeExtractorMatcherCacheSize is set to {}.", pipeExtractorMatcherCacheSize);
}
public int getPipeConnectorHandshakeTimeoutMs() {
@@ -826,12 +900,18 @@ public class CommonConfig {
}
public void setPipeConnectorHandshakeTimeoutMs(long pipeConnectorHandshakeTimeoutMs) {
+ final int fPipeConnectorHandshakeTimeoutMs = this.pipeConnectorHandshakeTimeoutMs;
try {
this.pipeConnectorHandshakeTimeoutMs = Math.toIntExact(pipeConnectorHandshakeTimeoutMs);
} catch (ArithmeticException e) {
this.pipeConnectorHandshakeTimeoutMs = Integer.MAX_VALUE;
logger.warn(
"Given pipe connector handshake timeout is too large, set to {} ms.", Integer.MAX_VALUE);
+ } finally {
+ if (fPipeConnectorHandshakeTimeoutMs != this.pipeConnectorHandshakeTimeoutMs) {
+ logger.info(
+ "pipeConnectorHandshakeTimeoutMs is set to {}.", fPipeConnectorHandshakeTimeoutMs);
+ }
}
}
@@ -840,12 +920,17 @@ public class CommonConfig {
}
public void setPipeConnectorTransferTimeoutMs(long pipeConnectorTransferTimeoutMs) {
+ final int fPipeConnectorTransferTimeoutMs = this.pipeConnectorTransferTimeoutMs;
try {
this.pipeConnectorTransferTimeoutMs = Math.toIntExact(pipeConnectorTransferTimeoutMs);
} catch (ArithmeticException e) {
this.pipeConnectorTransferTimeoutMs = Integer.MAX_VALUE;
logger.warn(
"Given pipe connector transfer timeout is too large, set to {} ms.", Integer.MAX_VALUE);
+ } finally {
+ if (fPipeConnectorTransferTimeoutMs != this.pipeConnectorTransferTimeoutMs) {
+ logger.info("pipeConnectorTransferTimeoutMs is set to {}.", pipeConnectorTransferTimeoutMs);
+ }
}
}
@@ -854,7 +939,11 @@ public class CommonConfig {
}
public void setPipeConnectorReadFileBufferSize(int pipeConnectorReadFileBufferSize) {
+ if (this.pipeConnectorReadFileBufferSize == pipeConnectorReadFileBufferSize) {
+ return;
+ }
this.pipeConnectorReadFileBufferSize = pipeConnectorReadFileBufferSize;
+ logger.info("pipeConnectorReadFileBufferSize is set to {}.", pipeConnectorReadFileBufferSize);
}
public boolean isPipeConnectorReadFileBufferMemoryControlEnabled() {
@@ -863,13 +952,27 @@ public class CommonConfig {
public void setIsPipeConnectorReadFileBufferMemoryControlEnabled(
boolean isPipeConnectorReadFileBufferMemoryControlEnabled) {
+ if (this.isPipeConnectorReadFileBufferMemoryControlEnabled
+ == isPipeConnectorReadFileBufferMemoryControlEnabled) {
+ return;
+ }
this.isPipeConnectorReadFileBufferMemoryControlEnabled =
isPipeConnectorReadFileBufferMemoryControlEnabled;
+ logger.info(
+ "isPipeConnectorReadFileBufferMemoryControlEnabled is set to {}.",
+ isPipeConnectorReadFileBufferMemoryControlEnabled);
}
public void setPipeConnectorRPCThriftCompressionEnabled(
boolean pipeConnectorRPCThriftCompressionEnabled) {
+ if (this.isPipeConnectorReadFileBufferMemoryControlEnabled
+ == pipeConnectorRPCThriftCompressionEnabled) {
+ return;
+ }
this.pipeConnectorRPCThriftCompressionEnabled = pipeConnectorRPCThriftCompressionEnabled;
+ logger.info(
+ "pipeConnectorRPCThriftCompressionEnabled is set to {}.",
+ pipeConnectorRPCThriftCompressionEnabled);
}
public boolean isPipeConnectorRPCThriftCompressionEnabled() {
@@ -878,8 +981,15 @@ public class CommonConfig {
public void setPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold(
int pipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold) {
+ if (this.pipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold
+ == pipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold) {
+ return;
+ }
this.pipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold =
pipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold;
+ logger.info(
+ "pipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold is set to {}.",
+ pipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold);
}
public int getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold() {
@@ -888,8 +998,15 @@ public class CommonConfig {
public void setPipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold(
int pipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold) {
+ if (this.pipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold
+ == pipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold) {
+ return;
+ }
this.pipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold =
pipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold;
+ logger.info(
+ "pipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold is set to {}.",
+ pipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold);
}
public int getPipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold() {
@@ -898,8 +1015,15 @@ public class CommonConfig {
public void setPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold(
int pipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold) {
+ if (this.pipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold
+ == pipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold) {
+ return;
+ }
this.pipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold =
pipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold;
+ logger.info(
+ "pipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold is set to {}.",
+ pipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold);
}
public int getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold() {
@@ -908,8 +1032,15 @@ public class CommonConfig {
public void setPipeAsyncConnectorMaxRetryExecutionTimeMsPerCall(
long pipeAsyncConnectorMaxRetryExecutionTimeMsPerCall) {
+ if (this.pipeAsyncConnectorMaxRetryExecutionTimeMsPerCall
+ == pipeAsyncConnectorMaxRetryExecutionTimeMsPerCall) {
+ return;
+ }
this.pipeAsyncConnectorMaxRetryExecutionTimeMsPerCall =
pipeAsyncConnectorMaxRetryExecutionTimeMsPerCall;
+ logger.info(
+ "pipeAsyncConnectorMaxRetryExecutionTimeMsPerCall is set to {}.",
+ pipeAsyncConnectorMaxRetryExecutionTimeMsPerCall);
}
public long getPipeAsyncConnectorMaxRetryExecutionTimeMsPerCall() {
@@ -921,7 +1052,11 @@ public class CommonConfig {
}
public void setPipeAsyncConnectorSelectorNumber(int pipeAsyncConnectorSelectorNumber) {
+ if (this.pipeAsyncConnectorSelectorNumber == pipeAsyncConnectorSelectorNumber) {
+ return;
+ }
this.pipeAsyncConnectorSelectorNumber = pipeAsyncConnectorSelectorNumber;
+ logger.info("pipeAsyncConnectorSelectorNumber is set to {}.", pipeAsyncConnectorSelectorNumber);
}
public int getPipeAsyncConnectorMaxClientNumber() {
@@ -929,7 +1064,12 @@ public class CommonConfig {
}
public void setPipeAsyncConnectorMaxClientNumber(int pipeAsyncConnectorMaxClientNumber) {
+ if (this.pipeAsyncConnectorMaxClientNumber == pipeAsyncConnectorMaxClientNumber) {
+ return;
+ }
this.pipeAsyncConnectorMaxClientNumber = pipeAsyncConnectorMaxClientNumber;
+ logger.info(
+ "pipeAsyncConnectorMaxClientNumber is set to {}.", pipeAsyncConnectorMaxClientNumber);
}
public boolean isSeperatedPipeHeartbeatEnabled() {
@@ -937,7 +1077,11 @@ public class CommonConfig {
}
public void setSeperatedPipeHeartbeatEnabled(boolean isSeperatedPipeHeartbeatEnabled) {
+ if (this.isSeperatedPipeHeartbeatEnabled == isSeperatedPipeHeartbeatEnabled) {
+ return;
+ }
this.isSeperatedPipeHeartbeatEnabled = isSeperatedPipeHeartbeatEnabled;
+ logger.info("isSeperatedPipeHeartbeatEnabled is set to {}.", isSeperatedPipeHeartbeatEnabled);
}
public int getPipeHeartbeatIntervalSecondsForCollectingPipeMeta() {
@@ -946,8 +1090,15 @@ public class CommonConfig {
public void setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(
int pipeHeartbeatIntervalSecondsForCollectingPipeMeta) {
+ if (this.pipeHeartbeatIntervalSecondsForCollectingPipeMeta
+ == pipeHeartbeatIntervalSecondsForCollectingPipeMeta) {
+ return;
+ }
this.pipeHeartbeatIntervalSecondsForCollectingPipeMeta =
pipeHeartbeatIntervalSecondsForCollectingPipeMeta;
+ logger.info(
+ "pipeHeartbeatIntervalSecondsForCollectingPipeMeta is set to {}.",
+ pipeHeartbeatIntervalSecondsForCollectingPipeMeta);
}
public long getPipeMetaSyncerInitialSyncDelayMinutes() {
@@ -955,7 +1106,13 @@ public class CommonConfig {
}
public void setPipeMetaSyncerInitialSyncDelayMinutes(long pipeMetaSyncerInitialSyncDelayMinutes) {
+ if (this.pipeMetaSyncerInitialSyncDelayMinutes == pipeMetaSyncerInitialSyncDelayMinutes) {
+ return;
+ }
this.pipeMetaSyncerInitialSyncDelayMinutes = pipeMetaSyncerInitialSyncDelayMinutes;
+ logger.info(
+ "pipeMetaSyncerInitialSyncDelayMinutes is set to {}.",
+ pipeMetaSyncerInitialSyncDelayMinutes);
}
public long getPipeMetaSyncerSyncIntervalMinutes() {
@@ -963,7 +1120,12 @@ public class CommonConfig {
}
public void setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSyncIntervalMinutes) {
+ if (this.pipeMetaSyncerSyncIntervalMinutes == pipeMetaSyncerSyncIntervalMinutes) {
+ return;
+ }
this.pipeMetaSyncerSyncIntervalMinutes = pipeMetaSyncerSyncIntervalMinutes;
+ logger.info(
+ "pipeMetaSyncerSyncIntervalMinutes is set to {}.", pipeMetaSyncerSyncIntervalMinutes);
}
public long getPipeMetaSyncerAutoRestartPipeCheckIntervalRound() {
@@ -972,8 +1134,15 @@ public class CommonConfig {
public void setPipeMetaSyncerAutoRestartPipeCheckIntervalRound(
long pipeMetaSyncerAutoRestartPipeCheckIntervalRound) {
+ if (this.pipeMetaSyncerAutoRestartPipeCheckIntervalRound
+ == pipeMetaSyncerAutoRestartPipeCheckIntervalRound) {
+ return;
+ }
this.pipeMetaSyncerAutoRestartPipeCheckIntervalRound =
pipeMetaSyncerAutoRestartPipeCheckIntervalRound;
+ logger.info(
+ "pipeMetaSyncerAutoRestartPipeCheckIntervalRound is set to {}.",
+ pipeMetaSyncerAutoRestartPipeCheckIntervalRound);
}
public boolean getPipeAutoRestartEnabled() {
@@ -981,7 +1150,11 @@ public class CommonConfig {
}
public void setPipeAutoRestartEnabled(boolean pipeAutoRestartEnabled) {
+ if (this.pipeAutoRestartEnabled == pipeAutoRestartEnabled) {
+ return;
+ }
this.pipeAutoRestartEnabled = pipeAutoRestartEnabled;
+ logger.info("pipeAutoRestartEnabled is set to {}.", pipeAutoRestartEnabled);
}
public long getPipeConnectorRetryIntervalMs() {
@@ -989,7 +1162,11 @@ public class CommonConfig {
}
public void setPipeConnectorRetryIntervalMs(long pipeConnectorRetryIntervalMs) {
+ if (this.pipeConnectorRetryIntervalMs == pipeConnectorRetryIntervalMs) {
+ return;
+ }
this.pipeConnectorRetryIntervalMs = pipeConnectorRetryIntervalMs;
+ logger.info("pipeConnectorRetryIntervalMs is set to {}", pipeConnectorRetryIntervalMs);
}
public int getPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount() {
@@ -998,8 +1175,15 @@ public class CommonConfig {
public void setPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount(
int pipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount) {
+ if (this.pipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount
+ == pipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount) {
+ return;
+ }
this.pipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount =
pipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount;
+ logger.info(
+ "pipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount is set to {}",
+ pipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount);
}
public long getPipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration() {
@@ -1008,8 +1192,15 @@ public class CommonConfig {
public void setPipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration(
long pipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration) {
+ if (this.pipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration
+ == pipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration) {
+ return;
+ }
this.pipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration =
pipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration;
+ logger.info(
+ "pipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration is set to {}",
+ pipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration);
}
public int getPipeSubtaskExecutorMaxThreadNum() {
@@ -1017,7 +1208,11 @@ public class CommonConfig {
}
public void setPipeSubtaskExecutorMaxThreadNum(int pipeSubtaskExecutorMaxThreadNum) {
+ if (this.pipeSubtaskExecutorMaxThreadNum == pipeSubtaskExecutorMaxThreadNum) {
+ return;
+ }
this.pipeSubtaskExecutorMaxThreadNum = pipeSubtaskExecutorMaxThreadNum;
+ logger.info("pipeSubtaskExecutorMaxThreadNum is set to {}.", pipeSubtaskExecutorMaxThreadNum);
}
public long getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs() {
@@ -1026,8 +1221,15 @@ public class CommonConfig {
public void setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(
long pipeSubtaskExecutorPendingQueueMaxBlockingTimeMs) {
+ if (this.pipeSubtaskExecutorPendingQueueMaxBlockingTimeMs
+ == pipeSubtaskExecutorPendingQueueMaxBlockingTimeMs) {
+ return;
+ }
this.pipeSubtaskExecutorPendingQueueMaxBlockingTimeMs =
pipeSubtaskExecutorPendingQueueMaxBlockingTimeMs;
+ logger.info(
+ "pipeSubtaskExecutorPendingQueueMaxBlockingTimeMs is set to {}",
+ pipeSubtaskExecutorPendingQueueMaxBlockingTimeMs);
}
public long getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds() {
@@ -1036,8 +1238,15 @@ public class CommonConfig {
public void setPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds(
long pipeSubtaskExecutorCronHeartbeatEventIntervalSeconds) {
+ if (this.pipeSubtaskExecutorCronHeartbeatEventIntervalSeconds
+ == pipeSubtaskExecutorCronHeartbeatEventIntervalSeconds) {
+ return;
+ }
this.pipeSubtaskExecutorCronHeartbeatEventIntervalSeconds =
pipeSubtaskExecutorCronHeartbeatEventIntervalSeconds;
+ logger.info(
+ "pipeSubtaskExecutorCronHeartbeatEventIntervalSeconds is set to {}.",
+ pipeSubtaskExecutorCronHeartbeatEventIntervalSeconds);
}
public long getPipeSubtaskExecutorForcedRestartIntervalMs() {
@@ -1046,7 +1255,14 @@ public class CommonConfig {
public void setPipeSubtaskExecutorForcedRestartIntervalMs(
long pipeSubtaskExecutorForcedRestartIntervalMs) {
+ if (this.pipeSubtaskExecutorForcedRestartIntervalMs
+ == pipeSubtaskExecutorForcedRestartIntervalMs) {
+ return;
+ }
this.pipeSubtaskExecutorForcedRestartIntervalMs = pipeSubtaskExecutorForcedRestartIntervalMs;
+ logger.info(
+ "pipeSubtaskExecutorForcedRestartIntervalMs is set to {}",
+ pipeSubtaskExecutorForcedRestartIntervalMs);
}
public int getPipeRealTimeQueuePollTsFileThreshold() {
@@ -1054,7 +1270,11 @@ public class CommonConfig {
}
public void setPipeRealTimeQueuePollTsFileThreshold(int pipeRealTimeQueuePollTsFileThreshold) {
+ if (this.pipeRealTimeQueuePollTsFileThreshold == pipeRealTimeQueuePollTsFileThreshold) {
+ return;
+ }
this.pipeRealTimeQueuePollTsFileThreshold = pipeRealTimeQueuePollTsFileThreshold;
+ logger.info("pipeRealTimeQueuePollTsFileThreshold is {}", pipeRealTimeQueuePollTsFileThreshold);
}
public int getPipeRealTimeQueuePollHistoricalTsFileThreshold() {
@@ -1063,12 +1283,23 @@ public class CommonConfig {
public void setPipeRealTimeQueuePollHistoricalTsFileThreshold(
int pipeRealTimeQueuePollHistoricalTsFileThreshold) {
+ if (this.pipeRealTimeQueuePollHistoricalTsFileThreshold
+ == pipeRealTimeQueuePollHistoricalTsFileThreshold) {
+ return;
+ }
this.pipeRealTimeQueuePollHistoricalTsFileThreshold =
pipeRealTimeQueuePollHistoricalTsFileThreshold;
+ logger.info(
+ "pipeRealTimeQueuePollHistoricalTsFileThreshold is set to {}",
+ pipeRealTimeQueuePollHistoricalTsFileThreshold);
}
public void setPipeAirGapReceiverEnabled(boolean pipeAirGapReceiverEnabled) {
+ if (pipeAirGapReceiverEnabled == this.pipeAirGapReceiverEnabled) {
+ return;
+ }
this.pipeAirGapReceiverEnabled = pipeAirGapReceiverEnabled;
+ logger.info("pipeAirGapReceiverEnabled is set to {}.", pipeAirGapReceiverEnabled);
}
public boolean getPipeAirGapReceiverEnabled() {
@@ -1076,7 +1307,11 @@ public class CommonConfig {
}
public void setPipeAirGapReceiverPort(int pipeAirGapReceiverPort) {
+ if (pipeAirGapReceiverPort == this.pipeAirGapReceiverPort) {
+ return;
+ }
this.pipeAirGapReceiverPort = pipeAirGapReceiverPort;
+ logger.info("pipeAirGapReceiverPort is set to {}.", pipeAirGapReceiverPort);
}
public int getPipeAirGapReceiverPort() {
@@ -1085,8 +1320,15 @@ public class CommonConfig {
public void setPipeReceiverLoginPeriodicVerificationIntervalMs(
long pipeReceiverLoginPeriodicVerificationIntervalMs) {
+ if (this.pipeReceiverLoginPeriodicVerificationIntervalMs
+ == pipeReceiverLoginPeriodicVerificationIntervalMs) {
+ return;
+ }
this.pipeReceiverLoginPeriodicVerificationIntervalMs =
pipeReceiverLoginPeriodicVerificationIntervalMs;
+ logger.info(
+ "pipeReceiverLoginPeriodicVerificationIntervalMs is set to {}",
+ pipeReceiverLoginPeriodicVerificationIntervalMs);
}
public long getPipeReceiverLoginPeriodicVerificationIntervalMs() {
@@ -1095,7 +1337,13 @@ public class CommonConfig {
public void setPipeReceiverActualToEstimatedMemoryRatio(
double pipeReceiverActualToEstimatedMemoryRatio) {
+ if (this.pipeReceiverActualToEstimatedMemoryRatio == pipeReceiverActualToEstimatedMemoryRatio) {
+ return;
+ }
this.pipeReceiverActualToEstimatedMemoryRatio = pipeReceiverActualToEstimatedMemoryRatio;
+ logger.info(
+ "pipeReceiverActualToEstimatedMemoryRatio is set to {}",
+ pipeReceiverActualToEstimatedMemoryRatio);
}
public double getPipeReceiverActualToEstimatedMemoryRatio() {
@@ -1108,8 +1356,15 @@ public class CommonConfig {
public void setPipeMaxAllowedHistoricalTsFilePerDataRegion(
int pipeMaxAllowedPendingTsFileEpochPerDataRegion) {
+ if (this.pipeMaxAllowedHistoricalTsFilePerDataRegion
+ == pipeMaxAllowedPendingTsFileEpochPerDataRegion) {
+ return;
+ }
this.pipeMaxAllowedHistoricalTsFilePerDataRegion =
pipeMaxAllowedPendingTsFileEpochPerDataRegion;
+ logger.info(
+ "pipeMaxAllowedHistoricalTsFilePerDataRegion is set to {}",
+ pipeMaxAllowedPendingTsFileEpochPerDataRegion);
}
public int getPipeMaxAllowedPendingTsFileEpochPerDataRegion() {
@@ -1118,7 +1373,14 @@ public class CommonConfig {
public void setPipeMaxAllowedPendingTsFileEpochPerDataRegion(
int pipeExtractorPendingQueueTsfileLimit) {
+ if (this.pipeMaxAllowedPendingTsFileEpochPerDataRegion
+ == pipeExtractorPendingQueueTsfileLimit) {
+ return;
+ }
this.pipeMaxAllowedPendingTsFileEpochPerDataRegion = pipeExtractorPendingQueueTsfileLimit;
+ logger.info(
+ "pipeMaxAllowedPendingTsFileEpochPerDataRegion is set to {}.",
+ pipeMaxAllowedPendingTsFileEpochPerDataRegion);
}
public int getPipeMaxAllowedPinnedMemTableCount() {
@@ -1126,7 +1388,12 @@ public class CommonConfig {
}
public void setPipeMaxAllowedPinnedMemTableCount(int pipeMaxAllowedPinnedMemTableCount) {
+ if (this.pipeMaxAllowedPinnedMemTableCount == pipeMaxAllowedPinnedMemTableCount) {
+ return;
+ }
this.pipeMaxAllowedPinnedMemTableCount = pipeMaxAllowedPinnedMemTableCount;
+ logger.info(
+ "pipeMaxAllowedPinnedMemTableCount is set to {}", pipeMaxAllowedPinnedMemTableCount);
}
public long getPipeMaxAllowedLinkedTsFileCount() {
@@ -1134,7 +1401,11 @@ public class CommonConfig {
}
public void setPipeMaxAllowedLinkedTsFileCount(long pipeMaxAllowedLinkedTsFileCount) {
+ if (this.pipeMaxAllowedLinkedTsFileCount == pipeMaxAllowedLinkedTsFileCount) {
+ return;
+ }
this.pipeMaxAllowedLinkedTsFileCount = pipeMaxAllowedLinkedTsFileCount;
+ logger.info("pipeMaxAllowedLinkedTsFileCount is set to {}", pipeMaxAllowedLinkedTsFileCount);
}
public float getPipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage() {
@@ -1143,8 +1414,15 @@ public class CommonConfig {
public void setPipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage(
float pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage) {
+ if (this.pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage
+ == pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage) {
+ return;
+ }
this.pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage =
pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage;
+ logger.info(
+ "pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage is set to {}",
+ pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage);
}
public long getPipeStuckRestartIntervalSeconds() {
@@ -1164,20 +1442,40 @@ public class CommonConfig {
}
public void setPipeStuckRestartIntervalSeconds(long pipeStuckRestartIntervalSeconds) {
+ if (this.pipeStuckRestartIntervalSeconds == pipeStuckRestartIntervalSeconds) {
+ return;
+ }
this.pipeStuckRestartIntervalSeconds = pipeStuckRestartIntervalSeconds;
+ logger.info("pipeStuckRestartIntervalSeconds is set to {}", pipeStuckRestartIntervalSeconds);
}
public void setPipeStuckRestartMinIntervalMs(long pipeStuckRestartMinIntervalMs) {
+ if (this.pipeStuckRestartMinIntervalMs == pipeStuckRestartMinIntervalMs) {
+ return;
+ }
this.pipeStuckRestartMinIntervalMs = pipeStuckRestartMinIntervalMs;
+ logger.info("pipeStuckRestartMinIntervalMs is set to {}", pipeStuckRestartMinIntervalMs);
}
public void setPipeEpochKeepTsFileAfterStuckRestartEnabled(
boolean pipeEpochKeepTsFileAfterStuckRestartEnabled) {
+ if (this.pipeEpochKeepTsFileAfterStuckRestartEnabled
+ == pipeEpochKeepTsFileAfterStuckRestartEnabled) {
+ return;
+ }
this.pipeEpochKeepTsFileAfterStuckRestartEnabled = pipeEpochKeepTsFileAfterStuckRestartEnabled;
+ logger.info(
+ "pipeEpochKeepTsFileAfterStuckRestartEnabled is set to {}",
+ pipeEpochKeepTsFileAfterStuckRestartEnabled);
}
public void setPipeStorageEngineFlushTimeIntervalMs(long pipeStorageEngineFlushTimeIntervalMs) {
+ if (this.pipeStorageEngineFlushTimeIntervalMs == pipeStorageEngineFlushTimeIntervalMs) {
+ return;
+ }
this.pipeStorageEngineFlushTimeIntervalMs = pipeStorageEngineFlushTimeIntervalMs;
+ logger.info(
+ "pipeStorageEngineFlushTimeIntervalMs is set to {}", pipeStorageEngineFlushTimeIntervalMs);
}
public int getPipeMetaReportMaxLogNumPerRound() {
@@ -1185,7 +1483,11 @@ public class CommonConfig {
}
public void setPipeMetaReportMaxLogNumPerRound(int pipeMetaReportMaxLogNumPerRound) {
+ if (this.pipeMetaReportMaxLogNumPerRound == pipeMetaReportMaxLogNumPerRound) {
+ return;
+ }
this.pipeMetaReportMaxLogNumPerRound = pipeMetaReportMaxLogNumPerRound;
+ logger.info("pipeMetaReportMaxLogNumPerRound is set to {}", pipeMetaReportMaxLogNumPerRound);
}
public int getPipeMetaReportMaxLogIntervalRounds() {
@@ -1193,7 +1495,12 @@ public class CommonConfig {
}
public void setPipeMetaReportMaxLogIntervalRounds(int pipeMetaReportMaxLogIntervalRounds) {
+ if (this.pipeMetaReportMaxLogIntervalRounds == pipeMetaReportMaxLogIntervalRounds) {
+ return;
+ }
this.pipeMetaReportMaxLogIntervalRounds = pipeMetaReportMaxLogIntervalRounds;
+ logger.info(
+ "pipeMetaReportMaxLogIntervalRounds is set to {}", pipeMetaReportMaxLogIntervalRounds);
}
public int getPipeTsFilePinMaxLogNumPerRound() {
@@ -1201,7 +1508,11 @@ public class CommonConfig {
}
public void setPipeTsFilePinMaxLogNumPerRound(int pipeTsFilePinMaxLogNumPerRound) {
+ if (this.pipeTsFilePinMaxLogNumPerRound == pipeTsFilePinMaxLogNumPerRound) {
+ return;
+ }
this.pipeTsFilePinMaxLogNumPerRound = pipeTsFilePinMaxLogNumPerRound;
+ logger.info("pipeTsFilePinMaxLogNumPerRound is set to {}", pipeTsFilePinMaxLogNumPerRound);
}
public int getPipeTsFilePinMaxLogIntervalRounds() {
@@ -1209,7 +1520,12 @@ public class CommonConfig {
}
public void setPipeTsFilePinMaxLogIntervalRounds(int pipeTsFilePinMaxLogIntervalRounds) {
+ if (this.pipeTsFilePinMaxLogIntervalRounds == pipeTsFilePinMaxLogIntervalRounds) {
+ return;
+ }
this.pipeTsFilePinMaxLogIntervalRounds = pipeTsFilePinMaxLogIntervalRounds;
+ logger.info(
+ "pipeTsFilePinMaxLogIntervalRounds is set to {}", pipeTsFilePinMaxLogIntervalRounds);
}
public int getPipeWalPinMaxLogNumPerRound() {
@@ -1217,7 +1533,11 @@ public class CommonConfig {
}
public void setPipeWalPinMaxLogNumPerRound(int pipeWalPinMaxLogNumPerRound) {
+ if (this.pipeWalPinMaxLogNumPerRound == pipeWalPinMaxLogNumPerRound) {
+ return;
+ }
this.pipeWalPinMaxLogNumPerRound = pipeWalPinMaxLogNumPerRound;
+ logger.info("pipeWalPinMaxLogNumPerRound is set to {}", pipeWalPinMaxLogNumPerRound);
}
public int getPipeWalPinMaxLogIntervalRounds() {
@@ -1225,7 +1545,11 @@ public class CommonConfig {
}
public void setPipeWalPinMaxLogIntervalRounds(int pipeWalPinMaxLogIntervalRounds) {
+ if (this.pipeWalPinMaxLogIntervalRounds == pipeWalPinMaxLogIntervalRounds) {
+ return;
+ }
this.pipeWalPinMaxLogIntervalRounds = pipeWalPinMaxLogIntervalRounds;
+ logger.info("pipeWalPinMaxLogIntervalRounds is set to {}", pipeWalPinMaxLogIntervalRounds);
}
public boolean getPipeMemoryManagementEnabled() {
@@ -1233,7 +1557,11 @@ public class CommonConfig {
}
public void setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled) {
+ if (this.pipeMemoryManagementEnabled == pipeMemoryManagementEnabled) {
+ return;
+ }
this.pipeMemoryManagementEnabled = pipeMemoryManagementEnabled;
+ logger.info("pipeMemoryManagementEnabled is set to {}", pipeMemoryManagementEnabled);
}
public long getPipeMemoryAllocateForTsFileSequenceReaderInBytes() {
@@ -1242,8 +1570,15 @@ public class CommonConfig {
public void setPipeMemoryAllocateForTsFileSequenceReaderInBytes(
long pipeMemoryAllocateForTsFileSequenceReaderInBytes) {
+ if (this.pipeMemoryAllocateForTsFileSequenceReaderInBytes
+ == pipeMemoryAllocateForTsFileSequenceReaderInBytes) {
+ return;
+ }
this.pipeMemoryAllocateForTsFileSequenceReaderInBytes =
pipeMemoryAllocateForTsFileSequenceReaderInBytes;
+ logger.info(
+ "pipeMemoryAllocateForTsFileSequenceReaderInBytes is set to {}",
+ pipeMemoryAllocateForTsFileSequenceReaderInBytes);
}
public long getPipeMemoryExpanderIntervalSeconds() {
@@ -1251,7 +1586,12 @@ public class CommonConfig {
}
public void setPipeMemoryExpanderIntervalSeconds(long pipeMemoryExpanderIntervalSeconds) {
+ if (this.pipeMemoryExpanderIntervalSeconds == pipeMemoryExpanderIntervalSeconds) {
+ return;
+ }
this.pipeMemoryExpanderIntervalSeconds = pipeMemoryExpanderIntervalSeconds;
+ logger.info(
+ "pipeMemoryExpanderIntervalSeconds is set to {}", pipeMemoryExpanderIntervalSeconds);
}
public long getPipeCheckMemoryEnoughIntervalMs() {
@@ -1259,7 +1599,11 @@ public class CommonConfig {
}
public void setPipeCheckMemoryEnoughIntervalMs(long pipeCheckMemoryEnoughIntervalMs) {
+ if (this.pipeCheckMemoryEnoughIntervalMs == pipeCheckMemoryEnoughIntervalMs) {
+ return;
+ }
this.pipeCheckMemoryEnoughIntervalMs = pipeCheckMemoryEnoughIntervalMs;
+ logger.info("pipeCheckMemoryEnoughIntervalMs is set to {}", pipeCheckMemoryEnoughIntervalMs);
}
public int getPipeMemoryAllocateMaxRetries() {
@@ -1267,7 +1611,11 @@ public class CommonConfig {
}
public void setPipeMemoryAllocateMaxRetries(int pipeMemoryAllocateMaxRetries) {
+ if (this.pipeMemoryAllocateMaxRetries == pipeMemoryAllocateMaxRetries) {
+ return;
+ }
this.pipeMemoryAllocateMaxRetries = pipeMemoryAllocateMaxRetries;
+ logger.info("pipeMemoryAllocateMaxRetries is set to {}", pipeMemoryAllocateMaxRetries);
}
public long getPipeMemoryAllocateRetryIntervalInMs() {
@@ -1275,7 +1623,12 @@ public class CommonConfig {
}
public void setPipeMemoryAllocateRetryIntervalInMs(long pipeMemoryAllocateRetryIntervalMs) {
+ if (this.pipeMemoryAllocateRetryIntervalMs == pipeMemoryAllocateRetryIntervalMs) {
+ return;
+ }
this.pipeMemoryAllocateRetryIntervalMs = pipeMemoryAllocateRetryIntervalMs;
+ logger.info(
+ "pipeMemoryAllocateRetryIntervalMs is set to {}", pipeMemoryAllocateRetryIntervalMs);
}
public long getPipeMemoryAllocateMinSizeInBytes() {
@@ -1283,7 +1636,11 @@ public class CommonConfig {
}
public void setPipeMemoryAllocateMinSizeInBytes(long pipeMemoryAllocateMinSizeInBytes) {
+ if (this.pipeMemoryAllocateMinSizeInBytes == pipeMemoryAllocateMinSizeInBytes) {
+ return;
+ }
this.pipeMemoryAllocateMinSizeInBytes = pipeMemoryAllocateMinSizeInBytes;
+ logger.info("pipeMemoryAllocateMinSizeInBytes is set to {}", pipeMemoryAllocateMinSizeInBytes);
}
public float getPipeLeaderCacheMemoryUsagePercentage() {
@@ -1291,7 +1648,12 @@ public class CommonConfig {
}
public void setPipeLeaderCacheMemoryUsagePercentage(float pipeLeaderCacheMemoryUsagePercentage) {
+ if (this.pipeLeaderCacheMemoryUsagePercentage == pipeLeaderCacheMemoryUsagePercentage) {
+ return;
+ }
this.pipeLeaderCacheMemoryUsagePercentage = pipeLeaderCacheMemoryUsagePercentage;
+ logger.info(
+ "pipeLeaderCacheMemoryUsagePercentage is set to {}", pipeLeaderCacheMemoryUsagePercentage);
}
public int getPipeMaxAlignedSeriesNumInOneBatch() {
@@ -1299,7 +1661,12 @@ public class CommonConfig {
}
public void setPipeMaxAlignedSeriesNumInOneBatch(int pipeMaxAlignedSeriesNumInOneBatch) {
+ if (this.pipeMaxAlignedSeriesNumInOneBatch == pipeMaxAlignedSeriesNumInOneBatch) {
+ return;
+ }
this.pipeMaxAlignedSeriesNumInOneBatch = pipeMaxAlignedSeriesNumInOneBatch;
+ logger.info(
+ "pipeMaxAlignedSeriesNumInOneBatch is set to {}", pipeMaxAlignedSeriesNumInOneBatch);
}
public long getPipeListeningQueueTransferSnapshotThreshold() {
@@ -1308,7 +1675,14 @@ public class CommonConfig {
public void setPipeListeningQueueTransferSnapshotThreshold(
long pipeListeningQueueTransferSnapshotThreshold) {
+ if (this.pipeListeningQueueTransferSnapshotThreshold
+ == pipeListeningQueueTransferSnapshotThreshold) {
+ return;
+ }
this.pipeListeningQueueTransferSnapshotThreshold = pipeListeningQueueTransferSnapshotThreshold;
+ logger.info(
+ "pipeListeningQueueTransferSnapshotThreshold is set to {}",
+ pipeListeningQueueTransferSnapshotThreshold);
}
public int getPipeSnapshotExecutionMaxBatchSize() {
@@ -1316,7 +1690,12 @@ public class CommonConfig {
}
public void setPipeSnapshotExecutionMaxBatchSize(int pipeSnapshotExecutionMaxBatchSize) {
+ if (this.pipeSnapshotExecutionMaxBatchSize == pipeSnapshotExecutionMaxBatchSize) {
+ return;
+ }
this.pipeSnapshotExecutionMaxBatchSize = pipeSnapshotExecutionMaxBatchSize;
+ logger.info(
+ "pipeSnapshotExecutionMaxBatchSize is set to {}", pipeSnapshotExecutionMaxBatchSize);
}
public long getPipeRemainingTimeCommitRateAutoSwitchSeconds() {
@@ -1325,8 +1704,15 @@ public class CommonConfig {
public void setPipeRemainingTimeCommitRateAutoSwitchSeconds(
long pipeRemainingTimeCommitRateAutoSwitchSeconds) {
+ if (this.pipeRemainingTimeCommitRateAutoSwitchSeconds
+ == pipeRemainingTimeCommitRateAutoSwitchSeconds) {
+ return;
+ }
this.pipeRemainingTimeCommitRateAutoSwitchSeconds =
pipeRemainingTimeCommitRateAutoSwitchSeconds;
+ logger.info(
+ "pipeRemainingTimeCommitRateAutoSwitchSeconds is set to {}",
+ pipeRemainingTimeCommitRateAutoSwitchSeconds);
}
public PipeRemainingTimeRateAverageTime getPipeRemainingTimeCommitRateAverageTime() {
@@ -1335,7 +1721,14 @@ public class CommonConfig {
public void setPipeRemainingTimeCommitRateAverageTime(
PipeRemainingTimeRateAverageTime pipeRemainingTimeCommitRateAverageTime) {
+ if (Objects.equals(
+ this.pipeRemainingTimeCommitRateAverageTime, pipeRemainingTimeCommitRateAverageTime)) {
+ return;
+ }
this.pipeRemainingTimeCommitRateAverageTime = pipeRemainingTimeCommitRateAverageTime;
+ logger.info(
+ "pipeRemainingTimeCommitRateAverageTime is set to {}",
+ pipeRemainingTimeCommitRateAverageTime);
}
public double getPipeTsFileScanParsingThreshold() {
@@ -1343,7 +1736,11 @@ public class CommonConfig {
}
public void setPipeTsFileScanParsingThreshold(double pipeTsFileScanParsingThreshold) {
+ if (this.pipeTsFileScanParsingThreshold == pipeTsFileScanParsingThreshold) {
+ return;
+ }
this.pipeTsFileScanParsingThreshold = pipeTsFileScanParsingThreshold;
+ logger.info("pipeTsFileScanParsingThreshold is set to {}", pipeTsFileScanParsingThreshold);
}
public double getPipeAllSinksRateLimitBytesPerSecond() {
@@ -1351,7 +1748,12 @@ public class CommonConfig {
}
public void setPipeAllSinksRateLimitBytesPerSecond(double pipeAllSinksRateLimitBytesPerSecond) {
+ if (this.pipeAllSinksRateLimitBytesPerSecond == pipeAllSinksRateLimitBytesPerSecond) {
+ return;
+ }
this.pipeAllSinksRateLimitBytesPerSecond = pipeAllSinksRateLimitBytesPerSecond;
+ logger.info(
+ "pipeAllSinksRateLimitBytesPerSecond is set to {}", pipeAllSinksRateLimitBytesPerSecond);
}
public int getRateLimiterHotReloadCheckIntervalMs() {
@@ -1359,7 +1761,12 @@ public class CommonConfig {
}
public void setRateLimiterHotReloadCheckIntervalMs(int rateLimiterHotReloadCheckIntervalMs) {
+ if (this.rateLimiterHotReloadCheckIntervalMs == rateLimiterHotReloadCheckIntervalMs) {
+ return;
+ }
this.rateLimiterHotReloadCheckIntervalMs = rateLimiterHotReloadCheckIntervalMs;
+ logger.info(
+ "rateLimiterHotReloadCheckIntervalMs is set to {}", rateLimiterHotReloadCheckIntervalMs);
}
public int getPipeConnectorRequestSliceThresholdBytes() {
@@ -1368,7 +1775,13 @@ public class CommonConfig {
public void setPipeConnectorRequestSliceThresholdBytes(
int pipeConnectorRequestSliceThresholdBytes) {
+ if (this.pipeConnectorRequestSliceThresholdBytes == pipeConnectorRequestSliceThresholdBytes) {
+ return;
+ }
this.pipeConnectorRequestSliceThresholdBytes = pipeConnectorRequestSliceThresholdBytes;
+ logger.info(
+ "pipeConnectorRequestSliceThresholdBytes is set to {}",
+ pipeConnectorRequestSliceThresholdBytes);
}
public long getTwoStageAggregateMaxCombinerLiveTimeInMs() {
@@ -1377,7 +1790,13 @@ public class CommonConfig {
public void setTwoStageAggregateMaxCombinerLiveTimeInMs(
long twoStageAggregateMaxCombinerLiveTimeInMs) {
+ if (this.twoStageAggregateMaxCombinerLiveTimeInMs == twoStageAggregateMaxCombinerLiveTimeInMs) {
+ return;
+ }
this.twoStageAggregateMaxCombinerLiveTimeInMs = twoStageAggregateMaxCombinerLiveTimeInMs;
+ logger.info(
+ "twoStageAggregateMaxCombinerLiveTimeInMs is set to {}",
+ twoStageAggregateMaxCombinerLiveTimeInMs);
}
public long getTwoStageAggregateDataRegionInfoCacheTimeInMs() {
@@ -1386,8 +1805,15 @@ public class CommonConfig {
public void setTwoStageAggregateDataRegionInfoCacheTimeInMs(
long twoStageAggregateDataRegionInfoCacheTimeInMs) {
+ if (this.twoStageAggregateDataRegionInfoCacheTimeInMs
+ == twoStageAggregateDataRegionInfoCacheTimeInMs) {
+ return;
+ }
this.twoStageAggregateDataRegionInfoCacheTimeInMs =
twoStageAggregateDataRegionInfoCacheTimeInMs;
+ logger.info(
+ "twoStageAggregateDataRegionInfoCacheTimeInMs is set to {}",
+ twoStageAggregateDataRegionInfoCacheTimeInMs);
}
public long getTwoStageAggregateSenderEndPointsCacheInMs() {
@@ -1396,7 +1822,14 @@ public class CommonConfig {
public void setTwoStageAggregateSenderEndPointsCacheInMs(
long twoStageAggregateSenderEndPointsCacheInMs) {
+ if (this.twoStageAggregateSenderEndPointsCacheInMs
+ == twoStageAggregateSenderEndPointsCacheInMs) {
+ return;
+ }
this.twoStageAggregateSenderEndPointsCacheInMs = twoStageAggregateSenderEndPointsCacheInMs;
+ logger.info(
+ "twoStageAggregateSenderEndPointsCacheInMs is set to {}",
+ twoStageAggregateSenderEndPointsCacheInMs);
}
public boolean getPipeEventReferenceTrackingEnabled() {
@@ -1404,7 +1837,12 @@ public class CommonConfig {
}
public void setPipeEventReferenceTrackingEnabled(boolean pipeEventReferenceTrackingEnabled) {
+ if (this.pipeEventReferenceTrackingEnabled == pipeEventReferenceTrackingEnabled) {
+ return;
+ }
this.pipeEventReferenceTrackingEnabled = pipeEventReferenceTrackingEnabled;
+ logger.info(
+ "pipeEventReferenceTrackingEnabled is set to {}", pipeEventReferenceTrackingEnabled);
}
public long getPipeEventReferenceEliminateIntervalSeconds() {
@@ -1413,7 +1851,14 @@ public class CommonConfig {
public void setPipeEventReferenceEliminateIntervalSeconds(
long pipeEventReferenceEliminateIntervalSeconds) {
+ if (this.pipeEventReferenceEliminateIntervalSeconds
+ == pipeEventReferenceEliminateIntervalSeconds) {
+ return;
+ }
this.pipeEventReferenceEliminateIntervalSeconds = pipeEventReferenceEliminateIntervalSeconds;
+ logger.info(
+ "pipeEventReferenceEliminateIntervalSeconds is set to {}",
+ pipeEventReferenceEliminateIntervalSeconds);
}
public float getSubscriptionCacheMemoryUsagePercentage() {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
index d57c129d0a6..ab9348f9de3 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.commons.conf;
import org.apache.iotdb.commons.enums.HandleSystemErrorStrategy;
-import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime;
+import org.apache.iotdb.commons.pipe.config.PipeDescriptor;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
@@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.Optional;
public class CommonDescriptor {
@@ -211,7 +210,7 @@ public class CommonDescriptor {
Integer.parseInt(
properties.getProperty("datanode_token_timeout", String.valueOf(3 * 60 * 1000))));
- loadPipeProps(properties);
+ PipeDescriptor.loadPipeProps(config, properties, false);
loadSubscriptionProps(properties);
config.setSchemaEngineMode(
@@ -256,487 +255,6 @@ public class CommonDescriptor {
loadBinaryAllocatorProps(properties);
}
- private void loadPipeProps(TrimProperties properties) {
- config.setPipeNonForwardingEventsProgressReportInterval(
- Integer.parseInt(
- properties.getProperty(
- "pipe_non_forwarding_events_progress_report_interval",
- Integer.toString(config.getPipeNonForwardingEventsProgressReportInterval()))));
-
- config.setPipeHardlinkBaseDirName(
- properties.getProperty("pipe_hardlink_base_dir_name", config.getPipeHardlinkBaseDirName()));
- config.setPipeHardlinkTsFileDirName(
- properties.getProperty(
- "pipe_hardlink_tsfile_dir_name", config.getPipeHardlinkTsFileDirName()));
- config.setPipeHardlinkWALDirName(
- properties.getProperty("pipe_hardlink_wal_dir_name", config.getPipeHardlinkWALDirName()));
- config.setPipeHardLinkWALEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_hardlink_wal_enabled",
- Boolean.toString(config.getPipeHardLinkWALEnabled()))));
- config.setPipeFileReceiverFsyncEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_file_receiver_fsync_enabled",
- Boolean.toString(config.getPipeFileReceiverFsyncEnabled()))));
-
- config.setPipeDataStructureTabletRowSize(
- Integer.parseInt(
- properties.getProperty(
- "pipe_data_structure_tablet_row_size",
- String.valueOf(config.getPipeDataStructureTabletRowSize()))));
- config.setPipeDataStructureTabletSizeInBytes(
- Integer.parseInt(
- properties.getProperty(
- "pipe_data_structure_tablet_size_in_bytes",
- String.valueOf(config.getPipeDataStructureTabletSizeInBytes()))));
- config.setPipeDataStructureTabletMemoryBlockAllocationRejectThreshold(
- Double.parseDouble(
- properties.getProperty(
- "pipe_data_structure_tablet_memory_block_allocation_reject_threshold",
- String.valueOf(
- config.getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold()))));
- config.setPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold(
- Double.parseDouble(
- properties.getProperty(
- "pipe_data_structure_ts_file_memory_block_allocation_reject_threshold",
- String.valueOf(
- config.getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold()))));
- config.setPipeTotalFloatingMemoryProportion(
- Double.parseDouble(
- properties.getProperty(
- "pipe_total_floating_memory_proportion",
- String.valueOf(config.getPipeTotalFloatingMemoryProportion()))));
-
- config.setPipeRealTimeQueuePollTsFileThreshold(
- Integer.parseInt(
- Optional.ofNullable(
- properties.getProperty("pipe_realtime_queue_poll_history_threshold"))
- .orElse(
- properties.getProperty(
- "pipe_realtime_queue_poll_tsfile_threshold",
- String.valueOf(config.getPipeRealTimeQueuePollTsFileThreshold())))));
- config.setPipeRealTimeQueuePollHistoricalTsFileThreshold(
- Integer.parseInt(
- properties.getProperty(
- "pipe_realtime_queue_poll_historical_tsfile_threshold",
- String.valueOf(config.getPipeRealTimeQueuePollHistoricalTsFileThreshold()))));
-
- int pipeSubtaskExecutorMaxThreadNum =
- Integer.parseInt(
- properties.getProperty(
- "pipe_subtask_executor_max_thread_num",
- Integer.toString(config.getPipeSubtaskExecutorMaxThreadNum())));
- if (pipeSubtaskExecutorMaxThreadNum > 0) {
- config.setPipeSubtaskExecutorMaxThreadNum(pipeSubtaskExecutorMaxThreadNum);
- }
- config.setPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount(
- Integer.parseInt(
- properties.getProperty(
- "pipe_subtask_executor_basic_check_point_interval_by_consumed_event_count",
- String.valueOf(
- config.getPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount()))));
- config.setPipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration(
- Long.parseLong(
- properties.getProperty(
- "pipe_subtask_executor_basic_check_point_interval_by_time_duration",
- String.valueOf(
- config.getPipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration()))));
- config.setPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(
- Long.parseLong(
- properties.getProperty(
- "pipe_subtask_executor_pending_queue_max_blocking_time_ms",
- String.valueOf(config.getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs()))));
- config.setPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds(
- Long.parseLong(
- properties.getProperty(
- "pipe_subtask_executor_cron_heartbeat_event_interval_seconds",
- String.valueOf(config.getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds()))));
- config.setPipeSubtaskExecutorForcedRestartIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "pipe_subtask_executor_forced_restart_interval_ms",
- String.valueOf(config.getPipeSubtaskExecutorForcedRestartIntervalMs()))));
-
- config.setPipeExtractorAssignerDisruptorRingBufferSize(
- Integer.parseInt(
- Optional.ofNullable(
- properties.getProperty("pipe_source_assigner_disruptor_ring_buffer_size"))
- .orElse(
- properties.getProperty(
- "pipe_extractor_assigner_disruptor_ring_buffer_size",
- String.valueOf(
- config.getPipeExtractorAssignerDisruptorRingBufferSize())))));
- config.setPipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes( // 1MB
- Integer.parseInt(
- Optional.ofNullable(
- properties.getProperty(
- "pipe_source_assigner_disruptor_ring_buffer_entry_size_in_bytes"))
- .orElse(
- properties.getProperty(
- "pipe_extractor_assigner_disruptor_ring_buffer_entry_size_in_bytes",
- String.valueOf(
- config
- .getPipeExtractorAssignerDisruptorRingBufferEntrySizeInBytes())))));
- config.setPipeExtractorMatcherCacheSize(
- Integer.parseInt(
- Optional.ofNullable(properties.getProperty("pipe_source_matcher_cache_size"))
- .orElse(
- properties.getProperty(
- "pipe_extractor_matcher_cache_size",
- String.valueOf(config.getPipeExtractorMatcherCacheSize())))));
-
- config.setPipeConnectorHandshakeTimeoutMs(
- Long.parseLong(
- Optional.ofNullable(properties.getProperty("pipe_sink_handshake_timeout_ms"))
- .orElse(
- properties.getProperty(
- "pipe_connector_handshake_timeout_ms",
- String.valueOf(config.getPipeConnectorHandshakeTimeoutMs())))));
- config.setPipeConnectorTransferTimeoutMs(
- Long.parseLong(
- Optional.ofNullable(properties.getProperty("pipe_sink_timeout_ms"))
- .orElse(
- properties.getProperty(
- "pipe_connector_timeout_ms",
- String.valueOf(config.getPipeConnectorTransferTimeoutMs())))));
- config.setPipeConnectorReadFileBufferSize(
- Integer.parseInt(
- Optional.ofNullable(properties.getProperty("pipe_sink_read_file_buffer_size"))
- .orElse(
- properties.getProperty(
- "pipe_connector_read_file_buffer_size",
- String.valueOf(config.getPipeConnectorReadFileBufferSize())))));
- config.setIsPipeConnectorReadFileBufferMemoryControlEnabled(
- Boolean.parseBoolean(
- Optional.ofNullable(properties.getProperty("pipe_sink_read_file_buffer_memory_control"))
- .orElse(
- properties.getProperty(
- "pipe_connector_read_file_buffer_memory_control",
- String.valueOf(
- config.isPipeConnectorReadFileBufferMemoryControlEnabled())))));
- config.setPipeConnectorRetryIntervalMs(
- Long.parseLong(
- Optional.ofNullable(properties.getProperty("pipe_sink_retry_interval_ms"))
- .orElse(
- properties.getProperty(
- "pipe_connector_retry_interval_ms",
- String.valueOf(config.getPipeConnectorRetryIntervalMs())))));
- config.setPipeConnectorRPCThriftCompressionEnabled(
- Boolean.parseBoolean(
- Optional.ofNullable(properties.getProperty("pipe_sink_rpc_thrift_compression_enabled"))
- .orElse(
- properties.getProperty(
- "pipe_connector_rpc_thrift_compression_enabled",
- String.valueOf(config.isPipeConnectorRPCThriftCompressionEnabled())))));
- config.setPipeAsyncConnectorMaxRetryExecutionTimeMsPerCall(
- Long.parseLong(
- Optional.ofNullable(
- properties.getProperty("pipe_async_sink_max_retry_execution_time_ms_per_call"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_max_retry_execution_time_ms_per_call",
- String.valueOf(
- config.getPipeAsyncConnectorMaxRetryExecutionTimeMsPerCall())))));
- config.setPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold(
- Integer.parseInt(
- Optional.ofNullable(
- properties.getProperty("pipe_async_sink_forced_retry_tsfile_event_queue_size"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_forced_retry_tsfile_event_queue_size",
- String.valueOf(
- config
- .getPipeAsyncConnectorForcedRetryTsFileEventQueueSizeThreshold())))));
- config.setPipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold(
- Integer.parseInt(
- Optional.ofNullable(
- properties.getProperty("pipe_async_sink_forced_retry_tablet_event_queue_size"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_forced_retry_tablet_event_queue_size",
- String.valueOf(
- config
- .getPipeAsyncConnectorForcedRetryTabletEventQueueSizeThreshold())))));
- config.setPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold(
- Integer.parseInt(
- Optional.ofNullable(
- properties.getProperty("pipe_async_sink_forced_retry_total_event_queue_size"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_forced_retry_total_event_queue_size",
- String.valueOf(
- config
- .getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold())))));
- int pipeAsyncConnectorSelectorNumber =
- Integer.parseInt(
- Optional.ofNullable(properties.getProperty("pipe_sink_selector_number"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_selector_number",
- String.valueOf(config.getPipeAsyncConnectorSelectorNumber()))));
- if (pipeAsyncConnectorSelectorNumber > 0) {
- config.setPipeAsyncConnectorSelectorNumber(pipeAsyncConnectorSelectorNumber);
- }
- int pipeAsyncConnectorMaxClientNumber =
- Integer.parseInt(
- Optional.ofNullable(properties.getProperty("pipe_sink_max_client_number"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_max_client_number",
- String.valueOf(config.getPipeAsyncConnectorMaxClientNumber()))));
- if (pipeAsyncConnectorMaxClientNumber > 0) {
- config.setPipeAsyncConnectorMaxClientNumber(pipeAsyncConnectorMaxClientNumber);
- }
- config.setPipeAllSinksRateLimitBytesPerSecond(
- Double.parseDouble(
- properties.getProperty(
- "pipe_all_sinks_rate_limit_bytes_per_second",
- String.valueOf(config.getPipeAllSinksRateLimitBytesPerSecond()))));
- config.setRateLimiterHotReloadCheckIntervalMs(
- Integer.parseInt(
- properties.getProperty(
- "rate_limiter_hot_reload_check_interval_ms",
- String.valueOf(config.getRateLimiterHotReloadCheckIntervalMs()))));
-
- config.setPipeConnectorRequestSliceThresholdBytes(
- Integer.parseInt(
- properties.getProperty(
- "pipe_connector_request_slice_threshold_bytes",
- String.valueOf(config.getPipeConnectorRequestSliceThresholdBytes()))));
-
- config.setSeperatedPipeHeartbeatEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_heartbeat_seperated_mode_enabled",
- String.valueOf(config.isSeperatedPipeHeartbeatEnabled()))));
- config.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(
- Integer.parseInt(
- properties.getProperty(
- "pipe_heartbeat_interval_seconds_for_collecting_pipe_meta",
- String.valueOf(config.getPipeHeartbeatIntervalSecondsForCollectingPipeMeta()))));
- config.setPipeMetaSyncerInitialSyncDelayMinutes(
- Long.parseLong(
- properties.getProperty(
- "pipe_meta_syncer_initial_sync_delay_minutes",
- String.valueOf(config.getPipeMetaSyncerInitialSyncDelayMinutes()))));
- config.setPipeMetaSyncerSyncIntervalMinutes(
- Long.parseLong(
- properties.getProperty(
- "pipe_meta_syncer_sync_interval_minutes",
- String.valueOf(config.getPipeMetaSyncerSyncIntervalMinutes()))));
- config.setPipeMetaSyncerAutoRestartPipeCheckIntervalRound(
- Long.parseLong(
- properties.getProperty(
- "pipe_meta_syncer_auto_restart_pipe_check_interval_round",
- String.valueOf(config.getPipeMetaSyncerAutoRestartPipeCheckIntervalRound()))));
- config.setPipeAutoRestartEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_auto_restart_enabled", String.valueOf(config.getPipeAutoRestartEnabled()))));
-
- config.setPipeAirGapReceiverEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_air_gap_receiver_enabled",
- Boolean.toString(config.getPipeAirGapReceiverEnabled()))));
- config.setPipeAirGapReceiverPort(
- Integer.parseInt(
- properties.getProperty(
- "pipe_air_gap_receiver_port",
- Integer.toString(config.getPipeAirGapReceiverPort()))));
-
- config.setPipeReceiverLoginPeriodicVerificationIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "pipe_receiver_login_periodic_verification_interval_ms",
- Long.toString(config.getPipeReceiverLoginPeriodicVerificationIntervalMs()))));
- config.setPipeReceiverActualToEstimatedMemoryRatio(
- Double.parseDouble(
- properties.getProperty(
- "pipe_receiver_actual_to_estimated_memory_ratio",
- Double.toString(config.getPipeReceiverActualToEstimatedMemoryRatio()))));
-
- config.setPipeMaxAllowedHistoricalTsFilePerDataRegion(
- Integer.parseInt(
- properties.getProperty(
- "pipe_max_allowed_historical_tsfile_per_data_region",
- String.valueOf(config.getPipeMaxAllowedHistoricalTsFilePerDataRegion()))));
- config.setPipeMaxAllowedPendingTsFileEpochPerDataRegion(
- Integer.parseInt(
- properties.getProperty(
- "pipe_max_allowed_pending_tsfile_epoch_per_data_region",
- String.valueOf(config.getPipeMaxAllowedPendingTsFileEpochPerDataRegion()))));
- config.setPipeMaxAllowedPinnedMemTableCount(
- Integer.parseInt(
- properties.getProperty(
- "pipe_max_allowed_pinned_memtable_count",
- String.valueOf(config.getPipeMaxAllowedPinnedMemTableCount()))));
- config.setPipeMaxAllowedLinkedTsFileCount(
- Long.parseLong(
- properties.getProperty(
- "pipe_max_allowed_linked_tsfile_count",
- String.valueOf(config.getPipeMaxAllowedLinkedTsFileCount()))));
- config.setPipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage(
- Float.parseFloat(
- properties.getProperty(
- "pipe_max_allowed_linked_deleted_tsfile_disk_usage_percentage",
- String.valueOf(config.getPipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage()))));
- config.setPipeStuckRestartIntervalSeconds(
- Long.parseLong(
- properties.getProperty(
- "pipe_stuck_restart_interval_seconds",
- String.valueOf(config.getPipeStuckRestartIntervalSeconds()))));
- config.setPipeStuckRestartMinIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "pipe_stuck_restart_min_interval_ms",
- String.valueOf(config.getPipeStuckRestartMinIntervalMs()))));
- config.setPipeEpochKeepTsFileAfterStuckRestartEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_epoch_keep_tsfile_after_stuck_restart_enabled",
- String.valueOf(config.isPipeEpochKeepTsFileAfterStuckRestartEnabled()))));
- config.setPipeStorageEngineFlushTimeIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "pipe_storage_engine_flush_time_interval_ms",
- String.valueOf(config.getPipeStorageEngineFlushTimeIntervalMs()))));
-
- config.setPipeMetaReportMaxLogNumPerRound(
- Integer.parseInt(
- properties.getProperty(
- "pipe_meta_report_max_log_num_per_round",
- String.valueOf(config.getPipeMetaReportMaxLogNumPerRound()))));
- config.setPipeMetaReportMaxLogIntervalRounds(
- Integer.parseInt(
- properties.getProperty(
- "pipe_meta_report_max_log_interval_rounds",
- String.valueOf(config.getPipeMetaReportMaxLogIntervalRounds()))));
- config.setPipeTsFilePinMaxLogNumPerRound(
- Integer.parseInt(
- properties.getProperty(
- "pipe_tsfile_pin_max_log_num_per_round",
- String.valueOf(config.getPipeTsFilePinMaxLogNumPerRound()))));
- config.setPipeTsFilePinMaxLogIntervalRounds(
- Integer.parseInt(
- properties.getProperty(
- "pipe_tsfile_pin_max_log_interval_rounds",
- String.valueOf(config.getPipeTsFilePinMaxLogIntervalRounds()))));
- config.setPipeWalPinMaxLogNumPerRound(
- Integer.parseInt(
- properties.getProperty(
- "pipe_wal_pin_max_log_num_per_round",
- String.valueOf(config.getPipeWalPinMaxLogNumPerRound()))));
- config.setPipeWalPinMaxLogIntervalRounds(
- Integer.parseInt(
- properties.getProperty(
- "pipe_wal_pin_max_log_interval_rounds",
- String.valueOf(config.getPipeWalPinMaxLogIntervalRounds()))));
-
- config.setPipeMemoryManagementEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_memory_management_enabled",
- String.valueOf(config.getPipeMemoryManagementEnabled()))));
- config.setPipeMemoryAllocateMaxRetries(
- Integer.parseInt(
- properties.getProperty(
- "pipe_memory_allocate_max_retries",
- String.valueOf(config.getPipeMemoryAllocateMaxRetries()))));
- config.setPipeMemoryAllocateRetryIntervalInMs(
- Long.parseLong(
- properties.getProperty(
- "pipe_memory_allocate_retry_interval_in_ms",
- String.valueOf(config.getPipeMemoryAllocateRetryIntervalInMs()))));
- config.setPipeMemoryAllocateMinSizeInBytes(
- Long.parseLong(
- properties.getProperty(
- "pipe_memory_allocate_min_size_in_bytes",
- String.valueOf(config.getPipeMemoryAllocateMinSizeInBytes()))));
- config.setPipeMemoryAllocateForTsFileSequenceReaderInBytes(
- Long.parseLong(
- properties.getProperty(
- "pipe_memory_allocate_for_tsfile_sequence_reader_in_bytes",
- String.valueOf(config.getPipeMemoryAllocateForTsFileSequenceReaderInBytes()))));
- config.setPipeMemoryExpanderIntervalSeconds(
- Long.parseLong(
- properties.getProperty(
- "pipe_memory_expander_interval_seconds",
- String.valueOf(config.getPipeMemoryExpanderIntervalSeconds()))));
- config.setPipeCheckMemoryEnoughIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "pipe_check_memory_enough_interval_ms",
- String.valueOf(config.getPipeCheckMemoryEnoughIntervalMs()))));
- config.setPipeLeaderCacheMemoryUsagePercentage(
- Float.parseFloat(
- properties.getProperty(
- "pipe_leader_cache_memory_usage_percentage",
- String.valueOf(config.getPipeLeaderCacheMemoryUsagePercentage()))));
- config.setPipeMaxAlignedSeriesNumInOneBatch(
- Integer.parseInt(
- properties.getProperty(
- "pipe_max_aligned_series_num_in_one_batch",
- String.valueOf(config.getPipeMaxAlignedSeriesNumInOneBatch()))));
- config.setPipeListeningQueueTransferSnapshotThreshold(
- Long.parseLong(
- properties.getProperty(
- "pipe_listening_queue_transfer_snapshot_threshold",
- String.valueOf(config.getPipeListeningQueueTransferSnapshotThreshold()))));
-
- config.setPipeSnapshotExecutionMaxBatchSize(
- Integer.parseInt(
- properties.getProperty(
- "pipe_snapshot_execution_max_batch_size",
- String.valueOf(config.getPipeSnapshotExecutionMaxBatchSize()))));
- config.setPipeRemainingTimeCommitRateAutoSwitchSeconds(
- Long.parseLong(
- properties.getProperty(
- "pipe_remaining_time_commit_rate_auto_switch_seconds",
- String.valueOf(config.getPipeRemainingTimeCommitRateAutoSwitchSeconds()))));
- config.setPipeRemainingTimeCommitRateAverageTime(
- PipeRemainingTimeRateAverageTime.valueOf(
- properties
- .getProperty(
- "pipe_remaining_time_commit_rate_average_time",
- String.valueOf(config.getPipeRemainingTimeCommitRateAverageTime()))
- .trim()));
- config.setPipeTsFileScanParsingThreshold(
- Double.parseDouble(
- properties.getProperty(
- "pipe_tsfile_scan_parsing_threshold",
- String.valueOf(config.getPipeTsFileScanParsingThreshold()))));
-
- config.setTwoStageAggregateMaxCombinerLiveTimeInMs(
- Long.parseLong(
- properties.getProperty(
- "two_stage_aggregate_max_combiner_live_time_in_ms",
- String.valueOf(config.getTwoStageAggregateMaxCombinerLiveTimeInMs()))));
- config.setTwoStageAggregateDataRegionInfoCacheTimeInMs(
- Long.parseLong(
- properties.getProperty(
- "two_stage_aggregate_data_region_info_cache_time_in_ms",
- String.valueOf(config.getTwoStageAggregateDataRegionInfoCacheTimeInMs()))));
- config.setTwoStageAggregateSenderEndPointsCacheInMs(
- Long.parseLong(
- properties.getProperty(
- "two_stage_aggregate_sender_end_points_cache_in_ms",
- String.valueOf(config.getTwoStageAggregateSenderEndPointsCacheInMs()))));
-
- config.setPipeEventReferenceTrackingEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_event_reference_tracking_enabled",
- String.valueOf(config.getPipeEventReferenceTrackingEnabled()))));
- config.setPipeEventReferenceEliminateIntervalSeconds(
- Long.parseLong(
- properties.getProperty(
- "pipe_event_reference_eliminate_interval_seconds",
- String.valueOf(config.getPipeEventReferenceEliminateIntervalSeconds()))));
- }
-
private void loadSubscriptionProps(TrimProperties properties) {
config.setSubscriptionCacheMemoryUsagePercentage(
Float.parseFloat(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java
index 82170661deb..aa93b093254 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java
@@ -36,8 +36,7 @@ public abstract class BlockingPendingQueue<E extends Event> {
private static final Logger LOGGER = LoggerFactory.getLogger(BlockingPendingQueue.class);
- private static final long MAX_BLOCKING_TIME_MS =
- PipeConfig.getInstance().getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs();
+ private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
protected final BlockingQueue<E> pendingQueue;
@@ -55,7 +54,10 @@ public abstract class BlockingPendingQueue<E extends Event> {
checkBeforeOffer(event);
try {
final boolean offered =
- pendingQueue.offer(event, MAX_BLOCKING_TIME_MS, TimeUnit.MILLISECONDS);
+ pendingQueue.offer(
+ event,
+ PIPE_CONFIG.getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(),
+ TimeUnit.MILLISECONDS);
if (offered) {
eventCounter.increaseEventCount(event);
}
@@ -98,7 +100,10 @@ public abstract class BlockingPendingQueue<E extends Event> {
public E waitedPoll() {
E event = null;
try {
- event = pendingQueue.poll(MAX_BLOCKING_TIME_MS, TimeUnit.MILLISECONDS);
+ event =
+ pendingQueue.poll(
+ PIPE_CONFIG.getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs(),
+ TimeUnit.MILLISECONDS);
eventCounter.decreaseEventCount(event);
} catch (final InterruptedException e) {
LOGGER.info("pending queue poll is interrupted.", e);
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskScheduler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskScheduler.java
index de47a8176ea..260077a73e8 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskScheduler.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/execution/PipeSubtaskScheduler.java
@@ -23,18 +23,15 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig;
public class PipeSubtaskScheduler {
+ private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
+
private final PipeSubtaskExecutor executor;
private boolean isFirstSchedule = true;
- private static final int BASIC_CHECKPOINT_INTERVAL_BY_CONSUMED_EVENT_COUNT =
- PipeConfig.getInstance().getPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount();
private int consumedEventCountCheckpointInterval;
private int consumedEventCount;
- // in ms
- private static final long BASIC_CHECKPOINT_INTERVAL_BY_TIME_DURATION =
- PipeConfig.getInstance().getPipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration();
private long timeDurationCheckpointInterval;
private long lastCheckTime;
@@ -73,13 +70,17 @@ public class PipeSubtaskScheduler {
Math.max(
1,
(int)
- (((float) BASIC_CHECKPOINT_INTERVAL_BY_CONSUMED_EVENT_COUNT / runningSubtaskNumber)
+ (((float)
+ PIPE_CONFIG
+ .getPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount()
+ / runningSubtaskNumber)
* corePoolSize));
timeDurationCheckpointInterval =
Math.max(
1,
(long)
- (((float) BASIC_CHECKPOINT_INTERVAL_BY_TIME_DURATION / runningSubtaskNumber)
+ (((float) PIPE_CONFIG.getPipeSubtaskExecutorBasicCheckPointIntervalByTimeDuration()
+ / runningSubtaskNumber)
* corePoolSize));
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
index a54fd96426a..0de34c0e08d 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeConfig.java
@@ -69,10 +69,13 @@ public class PipeConfig {
}
public double getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold() {
+ // To avoid too much parsed events causing OOM. If total tablet memory size exceeds this
+ // threshold, allocations of memory block for tablets will be rejected.
return COMMON_CONFIG.getPipeDataStructureTabletMemoryBlockAllocationRejectThreshold();
}
public double getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold() {
+ // Used to control the memory allocated for managing slice tsfile.
return COMMON_CONFIG.getPipeDataStructureTsFileMemoryBlockAllocationRejectThreshold();
}
@@ -87,7 +90,7 @@ public class PipeConfig {
}
public int getPipeRealTimeQueuePollHistoricalTsFileThreshold() {
- return COMMON_CONFIG.getPipeRealTimeQueuePollHistoricalTsFileThreshold();
+ return Math.max(COMMON_CONFIG.getPipeRealTimeQueuePollHistoricalTsFileThreshold(), 1);
}
/////////////////////////////// Subtask Executor ///////////////////////////////
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
similarity index 55%
copy from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
copy to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
index d57c129d0a6..a48469e7742 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java
@@ -17,264 +17,189 @@
* under the License.
*/
-package org.apache.iotdb.commons.conf;
+package org.apache.iotdb.commons.pipe.config;
-import org.apache.iotdb.commons.enums.HandleSystemErrorStrategy;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.ConfigurationFileUtils;
+import org.apache.iotdb.commons.conf.TrimProperties;
import org.apache.iotdb.commons.enums.PipeRemainingTimeRateAverageTime;
-import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
-import org.apache.iotdb.confignode.rpc.thrift.TGlobalConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
import java.io.IOException;
import java.util.Optional;
-public class CommonDescriptor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CommonDescriptor.class);
- private final CommonConfig config = new CommonConfig();
-
- private CommonDescriptor() {}
-
- public static CommonDescriptor getInstance() {
- return CommonDescriptorHolder.INSTANCE;
- }
-
- private static class CommonDescriptorHolder {
-
- private static final CommonDescriptor INSTANCE = new CommonDescriptor();
+public class PipeDescriptor {
- private CommonDescriptorHolder() {
- // empty constructor
+ public static void loadPipeProps(
+ CommonConfig config, TrimProperties properties, boolean isHotModify) throws IOException {
+ if (!isHotModify) {
+ loadPipeStaticConfig(config, properties);
}
- }
-
- public CommonConfig getConfig() {
- return config;
- }
- public String getConfDir() {
- // Check if a config-directory was specified first.
- String confString = System.getProperty(IoTDBConstant.IOTDB_CONF, null);
- // If it wasn't, check if a home directory was provided (This usually contains a config)
- if (confString == null) {
- confString = System.getProperty(IoTDBConstant.IOTDB_HOME, null);
- if (confString != null) {
- confString = confString + File.separatorChar + "conf";
- }
- }
- return confString;
- }
+ loadPipeInternalConfig(config, properties);
- public void initCommonConfigDir(String systemDir) {
- config.setUserFolder(systemDir + File.separator + "users");
- config.setRoleFolder(systemDir + File.separator + "roles");
- config.setProcedureWalFolder(systemDir + File.separator + "procedure");
+ loadPipeExternalConfig(config, properties, isHotModify);
}
- public void loadCommonProps(TrimProperties properties) throws IOException {
- config.setAuthorizerProvider(
- properties.getProperty("authorizer_provider_class", config.getAuthorizerProvider()).trim());
- // if using org.apache.iotdb.db.auth.authorizer.OpenIdAuthorizer, openID_url is needed.
- config.setOpenIdProviderUrl(
- properties.getProperty("openID_url", config.getOpenIdProviderUrl()).trim());
- config.setEncryptDecryptProvider(
- properties
- .getProperty(
- "iotdb_server_encrypt_decrypt_provider", config.getEncryptDecryptProvider())
- .trim());
-
- config.setEncryptDecryptProviderParameter(
+ public static void loadPipeStaticConfig(CommonConfig config, TrimProperties properties) {
+ config.setPipeHardlinkBaseDirName(
+ properties.getProperty("pipe_hardlink_base_dir_name", config.getPipeHardlinkBaseDirName()));
+ config.setPipeHardlinkTsFileDirName(
properties.getProperty(
- "iotdb_server_encrypt_decrypt_provider_parameter",
- config.getEncryptDecryptProviderParameter()));
-
- String[] tierTTLStr = new String[config.getTierTTLInMs().length];
- for (int i = 0; i < tierTTLStr.length; ++i) {
- tierTTLStr[i] = String.valueOf(config.getTierTTLInMs()[i]);
- }
- tierTTLStr =
- properties
- .getProperty("tier_ttl_in_ms", String.join(IoTDBConstant.TIER_SEPARATOR, tierTTLStr))
- .split(IoTDBConstant.TIER_SEPARATOR);
- long[] tierTTL = new long[tierTTLStr.length];
- for (int i = 0; i < tierTTL.length; ++i) {
- tierTTL[i] = Long.parseLong(tierTTLStr[i]);
- if (tierTTL[i] < 0) {
- tierTTL[i] = Long.MAX_VALUE;
- }
+ "pipe_hardlink_tsfile_dir_name", config.getPipeHardlinkTsFileDirName()));
+ config.setPipeHardlinkWALDirName(
+ properties.getProperty("pipe_hardlink_wal_dir_name", config.getPipeHardlinkWALDirName()));
+ config.setPipeHardLinkWALEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "pipe_hardlink_wal_enabled",
+ Boolean.toString(config.getPipeHardLinkWALEnabled()))));
+ int pipeSubtaskExecutorMaxThreadNum =
+ Integer.parseInt(
+ properties.getProperty(
+ "pipe_subtask_executor_max_thread_num",
+ Integer.toString(config.getPipeSubtaskExecutorMaxThreadNum())));
+ if (pipeSubtaskExecutorMaxThreadNum > 0) {
+ config.setPipeSubtaskExecutorMaxThreadNum(pipeSubtaskExecutorMaxThreadNum);
}
- config.setTierTTLInMs(tierTTL);
- config.setTTLCheckInterval(
+ config.setPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds(
Long.parseLong(
properties.getProperty(
- "ttl_check_interval", Long.toString(config.getTTLCheckInterval()))));
-
- config.setSyncDir(properties.getProperty("dn_sync_dir", config.getSyncDir()).trim());
-
- config.setWalDirs(
- properties
- .getProperty("dn_wal_dirs", String.join(",", config.getWalDirs()))
- .trim()
- .split(","));
+ "pipe_subtask_executor_cron_heartbeat_event_interval_seconds",
+ String.valueOf(config.getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds()))));
+ config.setSeperatedPipeHeartbeatEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "pipe_heartbeat_seperated_mode_enabled",
+ String.valueOf(config.isSeperatedPipeHeartbeatEnabled()))));
+ config.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(
+ Integer.parseInt(
+ properties.getProperty(
+ "pipe_heartbeat_interval_seconds_for_collecting_pipe_meta",
+ String.valueOf(config.getPipeHeartbeatIntervalSecondsForCollectingPipeMeta()))));
- config.setRpcThriftCompressionEnabled(
+ config.setPipeMetaSyncerInitialSyncDelayMinutes(
+ Long.parseLong(
+ properties.getProperty(
+ "pipe_meta_syncer_initial_sync_delay_minutes",
+ String.valueOf(config.getPipeMetaSyncerInitialSyncDelayMinutes()))));
+ config.setPipeMetaSyncerSyncIntervalMinutes(
+ Long.parseLong(
+ properties.getProperty(
+ "pipe_meta_syncer_sync_interval_minutes",
+ String.valueOf(config.getPipeMetaSyncerSyncIntervalMinutes()))));
+ config.setPipeMetaSyncerAutoRestartPipeCheckIntervalRound(
+ Long.parseLong(
+ properties.getProperty(
+ "pipe_meta_syncer_auto_restart_pipe_check_interval_round",
+ String.valueOf(config.getPipeMetaSyncerAutoRestartPipeCheckIntervalRound()))));
+ config.setPipeAutoRestartEnabled(
Boolean.parseBoolean(
- properties
- .getProperty(
- "cn_rpc_thrift_compression_enable",
- String.valueOf(config.isRpcThriftCompressionEnabled()))
- .trim()));
+ properties.getProperty(
+ "pipe_auto_restart_enabled", String.valueOf(config.getPipeAutoRestartEnabled()))));
- config.setCnConnectionTimeoutInMS(
+ config.setPipeAirGapReceiverEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "pipe_air_gap_receiver_enabled",
+ Boolean.toString(config.getPipeAirGapReceiverEnabled()))));
+ config.setPipeAirGapReceiverPort(
Integer.parseInt(
- properties
- .getProperty(
- "cn_connection_timeout_ms", String.valueOf(config.getCnConnectionTimeoutInMS()))
- .trim()));
+ properties.getProperty(
+ "pipe_air_gap_receiver_port",
+ Integer.toString(config.getPipeAirGapReceiverPort()))));
- config.setSelectorNumOfClientManager(
+ config.setPipeMetaReportMaxLogNumPerRound(
Integer.parseInt(
- properties
- .getProperty(
- "cn_selector_thread_nums_of_client_manager",
- String.valueOf(config.getSelectorNumOfClientManager()))
- .trim()));
-
- config.setMaxClientNumForEachNode(
+ properties.getProperty(
+ "pipe_meta_report_max_log_num_per_round",
+ String.valueOf(config.getPipeMetaReportMaxLogNumPerRound()))));
+ config.setPipeMetaReportMaxLogIntervalRounds(
Integer.parseInt(
- properties
- .getProperty(
- "cn_max_client_count_for_each_node_in_client_manager",
- String.valueOf(config.getMaxClientNumForEachNode()))
- .trim()));
-
- config.setDnConnectionTimeoutInMS(
+ properties.getProperty(
+ "pipe_meta_report_max_log_interval_rounds",
+ String.valueOf(config.getPipeMetaReportMaxLogIntervalRounds()))));
+ config.setPipeTsFilePinMaxLogNumPerRound(
Integer.parseInt(
- properties
- .getProperty(
- "dn_connection_timeout_ms", String.valueOf(config.getDnConnectionTimeoutInMS()))
- .trim()));
-
- config.setRpcThriftCompressionEnabled(
- Boolean.parseBoolean(
- properties
- .getProperty(
- "dn_rpc_thrift_compression_enable",
- String.valueOf(config.isRpcThriftCompressionEnabled()))
- .trim()));
-
- config.setSelectorNumOfClientManager(
+ properties.getProperty(
+ "pipe_tsfile_pin_max_log_num_per_round",
+ String.valueOf(config.getPipeTsFilePinMaxLogNumPerRound()))));
+ config.setPipeTsFilePinMaxLogIntervalRounds(
Integer.parseInt(
- properties
- .getProperty(
- "dn_selector_thread_nums_of_client_manager",
- String.valueOf(config.getSelectorNumOfClientManager()))
- .trim()));
-
- config.setMaxClientNumForEachNode(
+ properties.getProperty(
+ "pipe_tsfile_pin_max_log_interval_rounds",
+ String.valueOf(config.getPipeTsFilePinMaxLogIntervalRounds()))));
+ config.setPipeWalPinMaxLogNumPerRound(
Integer.parseInt(
- properties
- .getProperty(
- "dn_max_client_count_for_each_node_in_client_manager",
- String.valueOf(config.getMaxClientNumForEachNode()))
- .trim()));
-
- config.setHandleSystemErrorStrategy(
- HandleSystemErrorStrategy.valueOf(
- properties
- .getProperty(
- "handle_system_error", String.valueOf(config.getHandleSystemErrorStrategy()))
- .trim()));
-
- config.setDiskSpaceWarningThreshold(
- Double.parseDouble(
- properties
- .getProperty(
- "disk_space_warning_threshold",
- String.valueOf(config.getDiskSpaceWarningThreshold()))
- .trim()));
-
- config.setTimestampPrecision(
- properties.getProperty("timestamp_precision", config.getTimestampPrecision()).trim());
-
- config.setTimestampPrecisionCheckEnabled(
- Boolean.parseBoolean(
properties.getProperty(
- "timestamp_precision_check_enabled",
- String.valueOf(config.isTimestampPrecisionCheckEnabled()))));
-
- config.setDatanodeTokenTimeoutMS(
+ "pipe_wal_pin_max_log_num_per_round",
+ String.valueOf(config.getPipeWalPinMaxLogNumPerRound()))));
+ config.setPipeWalPinMaxLogIntervalRounds(
Integer.parseInt(
- properties.getProperty("datanode_token_timeout", String.valueOf(3 * 60 * 1000))));
-
- loadPipeProps(properties);
- loadSubscriptionProps(properties);
-
- config.setSchemaEngineMode(
- properties.getProperty("schema_engine_mode", String.valueOf(config.getSchemaEngineMode())));
+ properties.getProperty(
+ "pipe_wal_pin_max_log_interval_rounds",
+ String.valueOf(config.getPipeWalPinMaxLogIntervalRounds()))));
- config.setLastCacheEnable(
+ config.setPipeMemoryManagementEnabled(
Boolean.parseBoolean(
properties.getProperty(
- "enable_last_cache", Boolean.toString(config.isLastCacheEnable()))));
+ "pipe_memory_management_enabled",
+ String.valueOf(config.getPipeMemoryManagementEnabled()))));
- config.setTagAttributeTotalSize(
- Integer.parseInt(
+ config.setTwoStageAggregateMaxCombinerLiveTimeInMs(
+ Long.parseLong(
properties.getProperty(
- "tag_attribute_total_size", String.valueOf(config.getTagAttributeTotalSize()))));
-
- config.setTimePartitionOrigin(
+ "two_stage_aggregate_max_combiner_live_time_in_ms",
+ String.valueOf(config.getTwoStageAggregateMaxCombinerLiveTimeInMs()))));
+ config.setTwoStageAggregateDataRegionInfoCacheTimeInMs(
Long.parseLong(
properties.getProperty(
- "time_partition_origin", String.valueOf(config.getTimePartitionOrigin()))));
-
- config.setTimePartitionInterval(
+ "two_stage_aggregate_data_region_info_cache_time_in_ms",
+ String.valueOf(config.getTwoStageAggregateDataRegionInfoCacheTimeInMs()))));
+ config.setTwoStageAggregateSenderEndPointsCacheInMs(
Long.parseLong(
properties.getProperty(
- "time_partition_interval", String.valueOf(config.getTimePartitionInterval()))));
+ "two_stage_aggregate_sender_end_points_cache_in_ms",
+ String.valueOf(config.getTwoStageAggregateSenderEndPointsCacheInMs()))));
- config.setDatabaseLimitThreshold(
- Integer.parseInt(
+ config.setPipeEventReferenceTrackingEnabled(
+ Boolean.parseBoolean(
properties.getProperty(
- "database_limit_threshold", String.valueOf(config.getDatabaseLimitThreshold()))));
- config.setSeriesLimitThreshold(
+ "pipe_event_reference_tracking_enabled",
+ String.valueOf(config.getPipeEventReferenceTrackingEnabled()))));
+ config.setPipeEventReferenceEliminateIntervalSeconds(
Long.parseLong(
properties.getProperty(
- "cluster_timeseries_limit_threshold",
- String.valueOf(config.getSeriesLimitThreshold()))));
- config.setDeviceLimitThreshold(
+ "pipe_event_reference_eliminate_interval_seconds",
+ String.valueOf(config.getPipeEventReferenceEliminateIntervalSeconds()))));
+ config.setPipeListeningQueueTransferSnapshotThreshold(
Long.parseLong(
properties.getProperty(
- "cluster_device_limit_threshold",
- String.valueOf(config.getDeviceLimitThreshold()))));
+ "pipe_listening_queue_transfer_snapshot_threshold",
+ String.valueOf(config.getPipeListeningQueueTransferSnapshotThreshold()))));
- loadRetryProperties(properties);
- loadBinaryAllocatorProps(properties);
+ config.setPipeSnapshotExecutionMaxBatchSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "pipe_snapshot_execution_max_batch_size",
+ String.valueOf(config.getPipeSnapshotExecutionMaxBatchSize()))));
+ config.setPipeRemainingTimeCommitRateAverageTime(
+ PipeRemainingTimeRateAverageTime.valueOf(
+ properties
+ .getProperty(
+ "pipe_remaining_time_commit_rate_average_time",
+ String.valueOf(config.getPipeRemainingTimeCommitRateAverageTime()))
+ .trim()));
}
- private void loadPipeProps(TrimProperties properties) {
+ public static void loadPipeInternalConfig(CommonConfig config, TrimProperties properties)
+ throws IOException {
config.setPipeNonForwardingEventsProgressReportInterval(
Integer.parseInt(
properties.getProperty(
"pipe_non_forwarding_events_progress_report_interval",
Integer.toString(config.getPipeNonForwardingEventsProgressReportInterval()))));
- config.setPipeHardlinkBaseDirName(
- properties.getProperty("pipe_hardlink_base_dir_name", config.getPipeHardlinkBaseDirName()));
- config.setPipeHardlinkTsFileDirName(
- properties.getProperty(
- "pipe_hardlink_tsfile_dir_name", config.getPipeHardlinkTsFileDirName()));
- config.setPipeHardlinkWALDirName(
- properties.getProperty("pipe_hardlink_wal_dir_name", config.getPipeHardlinkWALDirName()));
- config.setPipeHardLinkWALEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_hardlink_wal_enabled",
- Boolean.toString(config.getPipeHardLinkWALEnabled()))));
config.setPipeFileReceiverFsyncEnabled(
Boolean.parseBoolean(
properties.getProperty(
@@ -322,15 +247,6 @@ public class CommonDescriptor {
properties.getProperty(
"pipe_realtime_queue_poll_historical_tsfile_threshold",
String.valueOf(config.getPipeRealTimeQueuePollHistoricalTsFileThreshold()))));
-
- int pipeSubtaskExecutorMaxThreadNum =
- Integer.parseInt(
- properties.getProperty(
- "pipe_subtask_executor_max_thread_num",
- Integer.toString(config.getPipeSubtaskExecutorMaxThreadNum())));
- if (pipeSubtaskExecutorMaxThreadNum > 0) {
- config.setPipeSubtaskExecutorMaxThreadNum(pipeSubtaskExecutorMaxThreadNum);
- }
config.setPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount(
Integer.parseInt(
properties.getProperty(
@@ -348,11 +264,6 @@ public class CommonDescriptor {
properties.getProperty(
"pipe_subtask_executor_pending_queue_max_blocking_time_ms",
String.valueOf(config.getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs()))));
- config.setPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds(
- Long.parseLong(
- properties.getProperty(
- "pipe_subtask_executor_cron_heartbeat_event_interval_seconds",
- String.valueOf(config.getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds()))));
config.setPipeSubtaskExecutorForcedRestartIntervalMs(
Long.parseLong(
properties.getProperty(
@@ -394,13 +305,6 @@ public class CommonDescriptor {
properties.getProperty(
"pipe_connector_handshake_timeout_ms",
String.valueOf(config.getPipeConnectorHandshakeTimeoutMs())))));
- config.setPipeConnectorTransferTimeoutMs(
- Long.parseLong(
- Optional.ofNullable(properties.getProperty("pipe_sink_timeout_ms"))
- .orElse(
- properties.getProperty(
- "pipe_connector_timeout_ms",
- String.valueOf(config.getPipeConnectorTransferTimeoutMs())))));
config.setPipeConnectorReadFileBufferSize(
Integer.parseInt(
Optional.ofNullable(properties.getProperty("pipe_sink_read_file_buffer_size"))
@@ -469,31 +373,6 @@ public class CommonDescriptor {
String.valueOf(
config
.getPipeAsyncConnectorForcedRetryTotalEventQueueSizeThreshold())))));
- int pipeAsyncConnectorSelectorNumber =
- Integer.parseInt(
- Optional.ofNullable(properties.getProperty("pipe_sink_selector_number"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_selector_number",
- String.valueOf(config.getPipeAsyncConnectorSelectorNumber()))));
- if (pipeAsyncConnectorSelectorNumber > 0) {
- config.setPipeAsyncConnectorSelectorNumber(pipeAsyncConnectorSelectorNumber);
- }
- int pipeAsyncConnectorMaxClientNumber =
- Integer.parseInt(
- Optional.ofNullable(properties.getProperty("pipe_sink_max_client_number"))
- .orElse(
- properties.getProperty(
- "pipe_async_connector_max_client_number",
- String.valueOf(config.getPipeAsyncConnectorMaxClientNumber()))));
- if (pipeAsyncConnectorMaxClientNumber > 0) {
- config.setPipeAsyncConnectorMaxClientNumber(pipeAsyncConnectorMaxClientNumber);
- }
- config.setPipeAllSinksRateLimitBytesPerSecond(
- Double.parseDouble(
- properties.getProperty(
- "pipe_all_sinks_rate_limit_bytes_per_second",
- String.valueOf(config.getPipeAllSinksRateLimitBytesPerSecond()))));
config.setRateLimiterHotReloadCheckIntervalMs(
Integer.parseInt(
properties.getProperty(
@@ -506,47 +385,6 @@ public class CommonDescriptor {
"pipe_connector_request_slice_threshold_bytes",
String.valueOf(config.getPipeConnectorRequestSliceThresholdBytes()))));
- config.setSeperatedPipeHeartbeatEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_heartbeat_seperated_mode_enabled",
- String.valueOf(config.isSeperatedPipeHeartbeatEnabled()))));
- config.setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(
- Integer.parseInt(
- properties.getProperty(
- "pipe_heartbeat_interval_seconds_for_collecting_pipe_meta",
- String.valueOf(config.getPipeHeartbeatIntervalSecondsForCollectingPipeMeta()))));
- config.setPipeMetaSyncerInitialSyncDelayMinutes(
- Long.parseLong(
- properties.getProperty(
- "pipe_meta_syncer_initial_sync_delay_minutes",
- String.valueOf(config.getPipeMetaSyncerInitialSyncDelayMinutes()))));
- config.setPipeMetaSyncerSyncIntervalMinutes(
- Long.parseLong(
- properties.getProperty(
- "pipe_meta_syncer_sync_interval_minutes",
- String.valueOf(config.getPipeMetaSyncerSyncIntervalMinutes()))));
- config.setPipeMetaSyncerAutoRestartPipeCheckIntervalRound(
- Long.parseLong(
- properties.getProperty(
- "pipe_meta_syncer_auto_restart_pipe_check_interval_round",
- String.valueOf(config.getPipeMetaSyncerAutoRestartPipeCheckIntervalRound()))));
- config.setPipeAutoRestartEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_auto_restart_enabled", String.valueOf(config.getPipeAutoRestartEnabled()))));
-
- config.setPipeAirGapReceiverEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_air_gap_receiver_enabled",
- Boolean.toString(config.getPipeAirGapReceiverEnabled()))));
- config.setPipeAirGapReceiverPort(
- Integer.parseInt(
- properties.getProperty(
- "pipe_air_gap_receiver_port",
- Integer.toString(config.getPipeAirGapReceiverPort()))));
-
config.setPipeReceiverLoginPeriodicVerificationIntervalMs(
Long.parseLong(
properties.getProperty(
@@ -604,42 +442,6 @@ public class CommonDescriptor {
"pipe_storage_engine_flush_time_interval_ms",
String.valueOf(config.getPipeStorageEngineFlushTimeIntervalMs()))));
- config.setPipeMetaReportMaxLogNumPerRound(
- Integer.parseInt(
- properties.getProperty(
- "pipe_meta_report_max_log_num_per_round",
- String.valueOf(config.getPipeMetaReportMaxLogNumPerRound()))));
- config.setPipeMetaReportMaxLogIntervalRounds(
- Integer.parseInt(
- properties.getProperty(
- "pipe_meta_report_max_log_interval_rounds",
- String.valueOf(config.getPipeMetaReportMaxLogIntervalRounds()))));
- config.setPipeTsFilePinMaxLogNumPerRound(
- Integer.parseInt(
- properties.getProperty(
- "pipe_tsfile_pin_max_log_num_per_round",
- String.valueOf(config.getPipeTsFilePinMaxLogNumPerRound()))));
- config.setPipeTsFilePinMaxLogIntervalRounds(
- Integer.parseInt(
- properties.getProperty(
- "pipe_tsfile_pin_max_log_interval_rounds",
- String.valueOf(config.getPipeTsFilePinMaxLogIntervalRounds()))));
- config.setPipeWalPinMaxLogNumPerRound(
- Integer.parseInt(
- properties.getProperty(
- "pipe_wal_pin_max_log_num_per_round",
- String.valueOf(config.getPipeWalPinMaxLogNumPerRound()))));
- config.setPipeWalPinMaxLogIntervalRounds(
- Integer.parseInt(
- properties.getProperty(
- "pipe_wal_pin_max_log_interval_rounds",
- String.valueOf(config.getPipeWalPinMaxLogIntervalRounds()))));
-
- config.setPipeMemoryManagementEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_memory_management_enabled",
- String.valueOf(config.getPipeMemoryManagementEnabled()))));
config.setPipeMemoryAllocateMaxRetries(
Integer.parseInt(
properties.getProperty(
@@ -680,250 +482,74 @@ public class CommonDescriptor {
properties.getProperty(
"pipe_max_aligned_series_num_in_one_batch",
String.valueOf(config.getPipeMaxAlignedSeriesNumInOneBatch()))));
- config.setPipeListeningQueueTransferSnapshotThreshold(
- Long.parseLong(
- properties.getProperty(
- "pipe_listening_queue_transfer_snapshot_threshold",
- String.valueOf(config.getPipeListeningQueueTransferSnapshotThreshold()))));
- config.setPipeSnapshotExecutionMaxBatchSize(
- Integer.parseInt(
- properties.getProperty(
- "pipe_snapshot_execution_max_batch_size",
- String.valueOf(config.getPipeSnapshotExecutionMaxBatchSize()))));
config.setPipeRemainingTimeCommitRateAutoSwitchSeconds(
Long.parseLong(
properties.getProperty(
"pipe_remaining_time_commit_rate_auto_switch_seconds",
String.valueOf(config.getPipeRemainingTimeCommitRateAutoSwitchSeconds()))));
- config.setPipeRemainingTimeCommitRateAverageTime(
- PipeRemainingTimeRateAverageTime.valueOf(
- properties
- .getProperty(
- "pipe_remaining_time_commit_rate_average_time",
- String.valueOf(config.getPipeRemainingTimeCommitRateAverageTime()))
- .trim()));
+
config.setPipeTsFileScanParsingThreshold(
Double.parseDouble(
properties.getProperty(
"pipe_tsfile_scan_parsing_threshold",
String.valueOf(config.getPipeTsFileScanParsingThreshold()))));
-
- config.setTwoStageAggregateMaxCombinerLiveTimeInMs(
- Long.parseLong(
- properties.getProperty(
- "two_stage_aggregate_max_combiner_live_time_in_ms",
- String.valueOf(config.getTwoStageAggregateMaxCombinerLiveTimeInMs()))));
- config.setTwoStageAggregateDataRegionInfoCacheTimeInMs(
- Long.parseLong(
- properties.getProperty(
- "two_stage_aggregate_data_region_info_cache_time_in_ms",
- String.valueOf(config.getTwoStageAggregateDataRegionInfoCacheTimeInMs()))));
- config.setTwoStageAggregateSenderEndPointsCacheInMs(
- Long.parseLong(
- properties.getProperty(
- "two_stage_aggregate_sender_end_points_cache_in_ms",
- String.valueOf(config.getTwoStageAggregateSenderEndPointsCacheInMs()))));
-
- config.setPipeEventReferenceTrackingEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "pipe_event_reference_tracking_enabled",
- String.valueOf(config.getPipeEventReferenceTrackingEnabled()))));
- config.setPipeEventReferenceEliminateIntervalSeconds(
- Long.parseLong(
- properties.getProperty(
- "pipe_event_reference_eliminate_interval_seconds",
- String.valueOf(config.getPipeEventReferenceEliminateIntervalSeconds()))));
}
- private void loadSubscriptionProps(TrimProperties properties) {
- config.setSubscriptionCacheMemoryUsagePercentage(
- Float.parseFloat(
- properties.getProperty(
- "subscription_cache_memory_usage_percentage",
- String.valueOf(config.getSubscriptionCacheMemoryUsagePercentage()))));
- config.setSubscriptionSubtaskExecutorMaxThreadNum(
- Integer.parseInt(
- properties.getProperty(
- "subscription_subtask_executor_max_thread_num",
- Integer.toString(config.getSubscriptionSubtaskExecutorMaxThreadNum()))));
+ public static void loadPipeExternalConfig(
+ CommonConfig config, TrimProperties properties, boolean isHotModify) throws IOException {
+ String value =
+ parserPipeConfig(
+ properties, "pipe_sink_timeout_ms", "pipe_connector_timeout_ms", isHotModify);
+ if (value != null) {
+ config.setPipeConnectorTransferTimeoutMs(Long.parseLong(value));
+ }
- config.setSubscriptionPrefetchTabletBatchMaxDelayInMs(
- Integer.parseInt(
- properties.getProperty(
- "subscription_prefetch_tablet_batch_max_delay_in_ms",
- String.valueOf(config.getSubscriptionPrefetchTabletBatchMaxDelayInMs()))));
- config.setSubscriptionPrefetchTabletBatchMaxSizeInBytes(
- Long.parseLong(
- properties.getProperty(
- "subscription_prefetch_tablet_batch_max_size_in_bytes",
- String.valueOf(config.getSubscriptionPrefetchTabletBatchMaxSizeInBytes()))));
- config.setSubscriptionPrefetchTsFileBatchMaxDelayInMs(
- Integer.parseInt(
- properties.getProperty(
- "subscription_prefetch_ts_file_batch_max_delay_in_ms",
- String.valueOf(config.getSubscriptionPrefetchTsFileBatchMaxDelayInMs()))));
- config.setSubscriptionPrefetchTsFileBatchMaxSizeInBytes(
- Long.parseLong(
- properties.getProperty(
- "subscription_prefetch_ts_file_batch_max_size_in_bytes",
- String.valueOf(config.getSubscriptionPrefetchTsFileBatchMaxSizeInBytes()))));
- config.setSubscriptionPollMaxBlockingTimeMs(
- Integer.parseInt(
- properties.getProperty(
- "subscription_poll_max_blocking_time_ms",
- String.valueOf(config.getSubscriptionPollMaxBlockingTimeMs()))));
- config.setSubscriptionDefaultTimeoutInMs(
- Integer.parseInt(
- properties.getProperty(
- "subscription_default_timeout_in_ms",
- String.valueOf(config.getSubscriptionDefaultTimeoutInMs()))));
- config.setSubscriptionLaunchRetryIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "subscription_launch_retry_interval_ms",
- String.valueOf(config.getSubscriptionLaunchRetryIntervalMs()))));
- config.setSubscriptionRecycleUncommittedEventIntervalMs(
- Integer.parseInt(
- properties.getProperty(
- "subscription_recycle_uncommitted_event_interval_ms",
- String.valueOf(config.getSubscriptionRecycleUncommittedEventIntervalMs()))));
- config.setSubscriptionReadFileBufferSize(
- Long.parseLong(
- properties.getProperty(
- "subscription_read_file_buffer_size",
- String.valueOf(config.getSubscriptionReadFileBufferSize()))));
- config.setSubscriptionReadTabletBufferSize(
- Long.parseLong(
- properties.getProperty(
- "subscription_read_tablet_buffer_size",
- String.valueOf(config.getSubscriptionReadTabletBufferSize()))));
- config.setSubscriptionTsFileDeduplicationWindowSeconds(
- Long.parseLong(
- properties.getProperty(
- "subscription_ts_file_deduplication_window_seconds",
- String.valueOf(config.getSubscriptionTsFileDeduplicationWindowSeconds()))));
- config.setSubscriptionCheckMemoryEnoughIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "subscription_check_memory_enough_interval_ms",
- String.valueOf(config.getSubscriptionCheckMemoryEnoughIntervalMs()))));
- config.setSubscriptionEstimatedInsertNodeTabletInsertionEventSize(
- Long.parseLong(
- properties.getProperty(
- "subscription_estimated_insert_node_tablet_insertion_event_size",
- String.valueOf(
- config.getSubscriptionEstimatedInsertNodeTabletInsertionEventSize()))));
- config.setSubscriptionEstimatedRawTabletInsertionEventSize(
- Long.parseLong(
- properties.getProperty(
- "subscription_estimated_raw_tablet_insertion_event_size",
- String.valueOf(config.getSubscriptionEstimatedRawTabletInsertionEventSize()))));
- config.setSubscriptionMaxAllowedEventCountInTabletBatch(
- Long.parseLong(
- properties.getProperty(
- "subscription_max_allowed_event_count_in_tablet_batch",
- String.valueOf(config.getSubscriptionMaxAllowedEventCountInTabletBatch()))));
- config.setSubscriptionLogManagerWindowSeconds(
- Long.parseLong(
- properties.getProperty(
- "subscription_log_manager_window_seconds",
- String.valueOf(config.getSubscriptionLogManagerWindowSeconds()))));
- config.setSubscriptionLogManagerBaseIntervalMs(
- Long.parseLong(
- properties.getProperty(
- "subscription_log_manager_base_interval_ms",
- String.valueOf(config.getSubscriptionLogManagerBaseIntervalMs()))));
+ value =
+ parserPipeConfig(
+ properties,
+ "pipe_sink_selector_number",
+ "pipe_async_connector_selector_number",
+ isHotModify);
+ if (value != null) {
+ config.setPipeAsyncConnectorSelectorNumber(Integer.parseInt(value));
+ }
- config.setSubscriptionPrefetchEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "subscription_prefetch_enabled",
- String.valueOf(config.getSubscriptionPrefetchEnabled()))));
- config.setSubscriptionPrefetchMemoryThreshold(
- Float.parseFloat(
- properties.getProperty(
- "subscription_prefetch_memory_threshold",
- String.valueOf(config.getSubscriptionPrefetchMemoryThreshold()))));
- config.setSubscriptionPrefetchMissingRateThreshold(
- Float.parseFloat(
- properties.getProperty(
- "subscription_prefetch_missing_rate_threshold",
- String.valueOf(config.getSubscriptionPrefetchMemoryThreshold()))));
- config.setSubscriptionPrefetchEventLocalCountThreshold(
- Integer.parseInt(
- properties.getProperty(
- "subscription_prefetch_event_local_count_threshold",
- String.valueOf(config.getSubscriptionPrefetchEventLocalCountThreshold()))));
- config.setSubscriptionPrefetchEventGlobalCountThreshold(
- Integer.parseInt(
- properties.getProperty(
- "subscription_prefetch_event_global_count_threshold",
- String.valueOf(config.getSubscriptionPrefetchEventGlobalCountThreshold()))));
+ value =
+ parserPipeConfig(
+ properties,
+ "pipe_sink_max_client_number",
+ "pipe_async_connector_max_client_number",
+ isHotModify);
+ if (value != null) {
+ config.setPipeAsyncConnectorMaxClientNumber(Integer.parseInt(value));
+ }
- config.setSubscriptionMetaSyncerInitialSyncDelayMinutes(
- Long.parseLong(
- properties.getProperty(
- "subscription_meta_syncer_initial_sync_delay_minutes",
- String.valueOf(config.getSubscriptionMetaSyncerInitialSyncDelayMinutes()))));
- config.setSubscriptionMetaSyncerSyncIntervalMinutes(
- Long.parseLong(
- properties.getProperty(
- "subscription_meta_syncer_sync_interval_minutes",
- String.valueOf(config.getSubscriptionMetaSyncerSyncIntervalMinutes()))));
+ value = parserPipeConfig(properties, "pipe_all_sinks_rate_limit_bytes_per_second", isHotModify);
+ if (value != null) {
+ config.setPipeAllSinksRateLimitBytesPerSecond(Double.parseDouble(value));
+ }
}
- public void loadRetryProperties(TrimProperties properties) throws IOException {
- config.setRemoteWriteMaxRetryDurationInMs(
- Long.parseLong(
- properties.getProperty(
- "write_request_remote_dispatch_max_retry_duration_in_ms",
- ConfigurationFileUtils.getConfigurationDefaultValue(
- "write_request_remote_dispatch_max_retry_duration_in_ms"))));
+ private static String parserPipeConfig(
+ final TrimProperties properties, final String key, final String otherKey, boolean isHotModify)
+ throws IOException {
- config.setRetryForUnknownErrors(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_retry_for_unknown_error",
- ConfigurationFileUtils.getConfigurationDefaultValue(
- "enable_retry_for_unknown_error"))));
- }
+ String value =
+ Optional.ofNullable(properties.getProperty(key)).orElse(properties.getProperty(otherKey));
- public void loadBinaryAllocatorProps(TrimProperties properties) {
- config.setEnableBinaryAllocator(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_binary_allocator", Boolean.toString(config.isEnableBinaryAllocator()))));
- config.setMinAllocateSize(
- Integer.parseInt(
- properties.getProperty(
- "small_blob_object", String.valueOf(config.getMinAllocateSize()))));
- config.setMaxAllocateSize(
- Integer.parseInt(
- properties.getProperty(
- "huge_blob_object", String.valueOf(config.getMaxAllocateSize()))));
- int arenaNum =
- Integer.parseInt(properties.getProperty("arena_num", String.valueOf(config.getArenaNum())));
- if (arenaNum > 0) {
- config.setArenaNum(arenaNum);
+ if (value == null && isHotModify) {
+ value =
+ Optional.ofNullable(ConfigurationFileUtils.getConfigurationDefaultValue(key))
+ .orElse(ConfigurationFileUtils.getConfigurationDefaultValue(otherKey));
}
- config.setLog2SizeClassGroup(
- Integer.parseInt(
- properties.getProperty(
- "log2_size_class_group", String.valueOf(config.getLog2SizeClassGroup()))));
+
+ return value;
}
- public void loadGlobalConfig(TGlobalConfig globalConfig) {
- config.setTimestampPrecision(globalConfig.timestampPrecision);
- config.setTimePartitionOrigin(
- CommonDateTimeUtils.convertMilliTimeWithPrecision(
- globalConfig.timePartitionOrigin, config.getTimestampPrecision()));
- config.setTimePartitionInterval(
- CommonDateTimeUtils.convertMilliTimeWithPrecision(
- globalConfig.timePartitionInterval, config.getTimestampPrecision()));
- config.setSchemaEngineMode(globalConfig.schemaEngineMode);
- config.setTagAttributeTotalSize(globalConfig.tagAttributeTotalSize);
- config.setDiskSpaceWarningThreshold(globalConfig.getDiskSpaceWarningThreshold());
+ private static String parserPipeConfig(
+ final TrimProperties properties, final String key, boolean isHotModify) throws IOException {
+ return Optional.ofNullable(properties.getProperty(key))
+ .orElse(isHotModify ? ConfigurationFileUtils.getConfigurationDefaultValue(key) : null);
}
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
index a8c65a93808..69e5664e127 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
@@ -74,12 +74,10 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver {
protected String username = CONNECTOR_IOTDB_USER_DEFAULT_VALUE;
protected String password = CONNECTOR_IOTDB_PASSWORD_DEFAULT_VALUE;
- private static final long LOGIN_PERIODIC_VERIFICATION_INTERVAL_MS =
- PipeConfig.getInstance().getPipeReceiverLoginPeriodicVerificationIntervalMs();
protected long lastSuccessfulLoginTime = Long.MIN_VALUE;
- private static final boolean IS_FSYNC_ENABLED =
- PipeConfig.getInstance().getPipeFileReceiverFsyncEnabled();
+ private static final PipeConfig PIPE_CONFIG = PipeConfig.getInstance();
+
private File writingFile;
private RandomAccessFile writingFileWriter;
@@ -320,9 +318,11 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver {
protected abstract String getClusterId();
protected boolean shouldLogin() {
- return LOGIN_PERIODIC_VERIFICATION_INTERVAL_MS >= 0
+ final long pipeReceiverLoginPeriodicVerificationIntervalMs =
+ PIPE_CONFIG.getPipeReceiverLoginPeriodicVerificationIntervalMs();
+ return pipeReceiverLoginPeriodicVerificationIntervalMs >= 0
&& lastSuccessfulLoginTime
- < System.currentTimeMillis() - LOGIN_PERIODIC_VERIFICATION_INTERVAL_MS;
+ < System.currentTimeMillis() - pipeReceiverLoginPeriodicVerificationIntervalMs;
}
protected TSStatus loginIfNecessary() {
@@ -449,7 +449,7 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver {
private void closeCurrentWritingFileWriter(final boolean fsyncBeforeClose) {
if (writingFileWriter != null) {
try {
- if (IS_FSYNC_ENABLED && fsyncBeforeClose) {
+ if (PIPE_CONFIG.getPipeFileReceiverFsyncEnabled() && fsyncBeforeClose) {
writingFileWriter.getFD().sync();
}
writingFileWriter.close();
@@ -549,7 +549,7 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver {
// Sync here is necessary to ensure that the data is written to the disk. Or data region may
// load the file before the data is written to the disk and cause unexpected behavior after
// system restart. (e.g., empty file in data region's data directory)
- if (IS_FSYNC_ENABLED) {
+ if (PIPE_CONFIG.getPipeFileReceiverFsyncEnabled()) {
writingFileWriter.getFD().sync();
}
// 1. The writing file writer must be closed, otherwise it may cause concurrent errors during
@@ -639,7 +639,7 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver {
// Sync here is necessary to ensure that the data is written to the disk. Or data region may
// load the file before the data is written to the disk and cause unexpected behavior after
// system restart. (e.g., empty file in data region's data directory)
- if (IS_FSYNC_ENABLED) {
+ if (PIPE_CONFIG.getPipeFileReceiverFsyncEnabled()) {
writingFileWriter.getFD().sync();
}
// 1. The writing file writer must be closed, otherwise it may cause concurrent errors during