You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2020/11/30 03:48:45 UTC
[iotdb] 01/01: add a strategy of closing file
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch close_file111
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit af942b939abc93f89b14e38e80e0710302f86ee3
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Nov 30 11:48:00 2020 +0800
add a strategy of closing file
---
.../resources/conf/iotdb-engine.properties | 3 +++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 16 +++++++++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 ++++
.../engine/storagegroup/StorageGroupProcessor.java | 1 +
.../db/engine/storagegroup/TsFileProcessor.java | 19 ++++++++++++++--
.../engine/storagegroup/TsFileProcessorInfo.java | 25 +++++++++++++++++++++-
6 files changed, 64 insertions(+), 4 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 2079a57..23b0f4b 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -241,6 +241,9 @@ reject_proportion=0.8
# If memory (in byte) of storage group increased more than this threshold, report to system. The default value is 16MB
storage_group_report_threshold=16777216
+#If the proportion of metadata size in a TsFile is over than this, the TsFile will be closed.
+metadata_file_size_proportion=0.1
+
# allowed max numbers of deduplicated path in one query
# it's just an advised value, the real limitation will be the smaller one between this and the one we calculated
max_deduplicated_path_num=1000
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 59e1ae2..09d93fb 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -169,7 +169,13 @@ public class IoTDBConfig {
/**
* When inserting rejected exceeds this, throw an exception
*/
- private int maxWaitingTimeWhenInsertBlockedInMs = 0;
+ private int maxWaitingTimeWhenInsertBlockedInMs = 0;
+
+ /**
+ * If the proportion of metadata size in a TsFile is over than this, close this TsFile
+ */
+ private double metadataFileSizeProportion = 0.1;
+
/**
* Is the write ahead log enable.
*/
@@ -2021,6 +2027,14 @@ public class IoTDBConfig {
public void setMaxWaitingTimeWhenInsertBlocked(int maxWaitingTimeWhenInsertBlocked) {
this.maxWaitingTimeWhenInsertBlockedInMs = maxWaitingTimeWhenInsertBlocked;
}
+
+ public double getMetadataFileSizeProportion() {
+ return metadataFileSizeProportion;
+ }
+
+ public void setMetadataFileSizeProportion(double metadataFileSizeProportion) {
+ this.metadataFileSizeProportion = metadataFileSizeProportion;
+ }
public int getFrequencyIntervalInMinute() {
return frequencyIntervalInMinute;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index d7a7058..7e3c86c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -291,6 +291,10 @@ public class IoTDBDescriptor {
.getProperty("max_waiting_time_when_insert_blocked",
Integer.toString(conf.getMaxWaitingTimeWhenInsertBlocked()))));
+ conf.setMetadataFileSizeProportion(Double.parseDouble(properties
+ .getProperty("metadata_file_size_proportion",
+ Double.toString(conf.getMetadataFileSizeProportion()))));
+
conf.setEstimatedSeriesSize(Integer.parseInt(properties
.getProperty("estimated_series_size",
Integer.toString(conf.getEstimatedSeriesSize()))));
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 5cd831a..3a67674 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -631,6 +631,7 @@ public class StorageGroupProcessor {
}
}
tsFileProcessor.getTsFileProcessorInfo().addTSPMemCost(chunkMetadataSize);
+ tsFileProcessor.getTsFileProcessorInfo().releaseMetadataMemCost(chunkMetadataSize);
}
}
tsFileManagement.add(tsFileResource, isSeq);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 1007948..6b9b555 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -286,6 +286,7 @@ public class TsFileProcessor {
memTableIncrement += textDataIncrement;
storageGroupInfo.addStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.addTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.addMetadataMemCost(chunkMetadataIncrement);
if (storageGroupInfo.needToReportToSystem()) {
SystemInfo.getInstance().reportStorageGroupStatus(storageGroupInfo);
try {
@@ -293,6 +294,7 @@ public class TsFileProcessor {
} catch (WriteProcessException e) {
storageGroupInfo.releaseStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.releaseTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.releaseMetadataMemCost(chunkMetadataIncrement);
SystemInfo.getInstance().resetStorageGroupStatus(storageGroupInfo, false);
throw e;
}
@@ -350,6 +352,7 @@ public class TsFileProcessor {
memTableIncrement += textDataIncrement;
storageGroupInfo.addStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.addTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.addMetadataMemCost(chunkMetadataIncrement);
if (storageGroupInfo.needToReportToSystem()) {
SystemInfo.getInstance().reportStorageGroupStatus(storageGroupInfo);
try {
@@ -357,6 +360,7 @@ public class TsFileProcessor {
} catch (WriteProcessException e) {
storageGroupInfo.releaseStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.releaseTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.releaseMetadataMemCost(chunkMetadataIncrement);
SystemInfo.getInstance().resetStorageGroupStatus(storageGroupInfo, false);
throw e;
}
@@ -445,11 +449,22 @@ public class TsFileProcessor {
public boolean shouldClose() {
long fileSize = tsFileResource.getTsFileSize();
- long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig()
- .getTsFileSizeThreshold();
+ long fileSizeThreshold = config.getTsFileSizeThreshold();
if (fileSize >= fileSizeThreshold) {
logger.info("{} fileSize {} >= fileSizeThreshold {}", tsFileResource.getTsFilePath(),
fileSize, fileSizeThreshold);
+ return true;
+ }
+ if (enableMemControl) {
+ double metadataFileSizeProportion = config.getMetadataFileSizeProportion();
+ if ((double) tsFileProcessorInfo.getMetadataMemCost() / fileSize
+ >= metadataFileSizeProportion) {
+ logger.info("{} MetadataSize {} / FileSize {} >= MetadataFileSizeProportion {}",
+ tsFileResource.getTsFilePath(),
+ tsFileProcessorInfo.getMetadataMemCost(),
+ fileSize, metadataFileSizeProportion);
+ return true;
+ }
}
return fileSize >= fileSizeThreshold;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java
index 417168e..b7b418a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java
@@ -35,6 +35,10 @@ public class TsFileProcessorInfo {
*/
private long memCost;
+ /**
+ * memory occupation of ChunkMetadata
+ */
+ private long metadataMemCost = 0L;
public TsFileProcessorInfo(StorageGroupInfo storageGroupInfo) {
this.storageGroupInfo = storageGroupInfo;
@@ -58,10 +62,29 @@ public class TsFileProcessorInfo {
}
/**
+ * called in each insert
+ */
+ public void addMetadataMemCost(long cost) {
+ metadataMemCost += cost;
+ }
+
+ /**
+ * called when meet exception
+ */
+ public void releaseMetadataMemCost(long cost) {
+ metadataMemCost -= cost;
+ }
+
+ public long getMetadataMemCost() {
+ return metadataMemCost;
+ }
+
+ /**
* called when closing TSP
*/
public void clear() {
storageGroupInfo.releaseStorageGroupMemCost(memCost);
- memCost = 0;
+ memCost = 0L;
+ metadataMemCost = 0L;
}
}