You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2020/11/30 03:48:44 UTC
[iotdb] branch close_file111 created (now af942b9)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch close_file111
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at af942b9 add a strategy of closing file
This branch includes the following new commits:
new af942b9 add a strategy of closing file
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: add a strategy of closing file
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch close_file111
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit af942b939abc93f89b14e38e80e0710302f86ee3
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Nov 30 11:48:00 2020 +0800
add a strategy of closing file
---
.../resources/conf/iotdb-engine.properties | 3 +++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 16 +++++++++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 ++++
.../engine/storagegroup/StorageGroupProcessor.java | 1 +
.../db/engine/storagegroup/TsFileProcessor.java | 19 ++++++++++++++--
.../engine/storagegroup/TsFileProcessorInfo.java | 25 +++++++++++++++++++++-
6 files changed, 64 insertions(+), 4 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 2079a57..23b0f4b 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -241,6 +241,9 @@ reject_proportion=0.8
# If memory (in byte) of storage group increased more than this threshold, report to system. The default value is 16MB
storage_group_report_threshold=16777216
+#If the proportion of metadata size in a TsFile is over than this, the TsFile will be closed.
+metadata_file_size_proportion=0.1
+
# allowed max numbers of deduplicated path in one query
# it's just an advised value, the real limitation will be the smaller one between this and the one we calculated
max_deduplicated_path_num=1000
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 59e1ae2..09d93fb 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -169,7 +169,13 @@ public class IoTDBConfig {
/**
* When inserting rejected exceeds this, throw an exception
*/
- private int maxWaitingTimeWhenInsertBlockedInMs = 0;
+ private int maxWaitingTimeWhenInsertBlockedInMs = 0;
+
+ /**
+ * If the proportion of metadata size in a TsFile is over than this, close this TsFile
+ */
+ private double metadataFileSizeProportion = 0.1;
+
/**
* Is the write ahead log enable.
*/
@@ -2021,6 +2027,14 @@ public class IoTDBConfig {
public void setMaxWaitingTimeWhenInsertBlocked(int maxWaitingTimeWhenInsertBlocked) {
this.maxWaitingTimeWhenInsertBlockedInMs = maxWaitingTimeWhenInsertBlocked;
}
+
+ public double getMetadataFileSizeProportion() {
+ return metadataFileSizeProportion;
+ }
+
+ public void setMetadataFileSizeProportion(double metadataFileSizeProportion) {
+ this.metadataFileSizeProportion = metadataFileSizeProportion;
+ }
public int getFrequencyIntervalInMinute() {
return frequencyIntervalInMinute;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index d7a7058..7e3c86c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -291,6 +291,10 @@ public class IoTDBDescriptor {
.getProperty("max_waiting_time_when_insert_blocked",
Integer.toString(conf.getMaxWaitingTimeWhenInsertBlocked()))));
+ conf.setMetadataFileSizeProportion(Double.parseDouble(properties
+ .getProperty("metadata_file_size_proportion",
+ Double.toString(conf.getMetadataFileSizeProportion()))));
+
conf.setEstimatedSeriesSize(Integer.parseInt(properties
.getProperty("estimated_series_size",
Integer.toString(conf.getEstimatedSeriesSize()))));
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 5cd831a..3a67674 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -631,6 +631,7 @@ public class StorageGroupProcessor {
}
}
tsFileProcessor.getTsFileProcessorInfo().addTSPMemCost(chunkMetadataSize);
+ tsFileProcessor.getTsFileProcessorInfo().releaseMetadataMemCost(chunkMetadataSize);
}
}
tsFileManagement.add(tsFileResource, isSeq);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 1007948..6b9b555 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -286,6 +286,7 @@ public class TsFileProcessor {
memTableIncrement += textDataIncrement;
storageGroupInfo.addStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.addTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.addMetadataMemCost(chunkMetadataIncrement);
if (storageGroupInfo.needToReportToSystem()) {
SystemInfo.getInstance().reportStorageGroupStatus(storageGroupInfo);
try {
@@ -293,6 +294,7 @@ public class TsFileProcessor {
} catch (WriteProcessException e) {
storageGroupInfo.releaseStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.releaseTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.releaseMetadataMemCost(chunkMetadataIncrement);
SystemInfo.getInstance().resetStorageGroupStatus(storageGroupInfo, false);
throw e;
}
@@ -350,6 +352,7 @@ public class TsFileProcessor {
memTableIncrement += textDataIncrement;
storageGroupInfo.addStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.addTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.addMetadataMemCost(chunkMetadataIncrement);
if (storageGroupInfo.needToReportToSystem()) {
SystemInfo.getInstance().reportStorageGroupStatus(storageGroupInfo);
try {
@@ -357,6 +360,7 @@ public class TsFileProcessor {
} catch (WriteProcessException e) {
storageGroupInfo.releaseStorageGroupMemCost(memTableIncrement);
tsFileProcessorInfo.releaseTSPMemCost(unsealedResourceIncrement + chunkMetadataIncrement);
+ tsFileProcessorInfo.releaseMetadataMemCost(chunkMetadataIncrement);
SystemInfo.getInstance().resetStorageGroupStatus(storageGroupInfo, false);
throw e;
}
@@ -445,11 +449,22 @@ public class TsFileProcessor {
public boolean shouldClose() {
long fileSize = tsFileResource.getTsFileSize();
- long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig()
- .getTsFileSizeThreshold();
+ long fileSizeThreshold = config.getTsFileSizeThreshold();
if (fileSize >= fileSizeThreshold) {
logger.info("{} fileSize {} >= fileSizeThreshold {}", tsFileResource.getTsFilePath(),
fileSize, fileSizeThreshold);
+ return true;
+ }
+ if (enableMemControl) {
+ double metadataFileSizeProportion = config.getMetadataFileSizeProportion();
+ if ((double) tsFileProcessorInfo.getMetadataMemCost() / fileSize
+ >= metadataFileSizeProportion) {
+ logger.info("{} MetadataSize {} / FileSize {} >= MetadataFileSizeProportion {}",
+ tsFileResource.getTsFilePath(),
+ tsFileProcessorInfo.getMetadataMemCost(),
+ fileSize, metadataFileSizeProportion);
+ return true;
+ }
}
return fileSize >= fileSizeThreshold;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java
index 417168e..b7b418a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java
@@ -35,6 +35,10 @@ public class TsFileProcessorInfo {
*/
private long memCost;
+ /**
+ * memory occupation of ChunkMetadata
+ */
+ private long metadataMemCost = 0L;
public TsFileProcessorInfo(StorageGroupInfo storageGroupInfo) {
this.storageGroupInfo = storageGroupInfo;
@@ -58,10 +62,29 @@ public class TsFileProcessorInfo {
}
/**
+ * called in each insert
+ */
+ public void addMetadataMemCost(long cost) {
+ metadataMemCost += cost;
+ }
+
+ /**
+ * called when meet exception
+ */
+ public void releaseMetadataMemCost(long cost) {
+ metadataMemCost -= cost;
+ }
+
+ public long getMetadataMemCost() {
+ return metadataMemCost;
+ }
+
+ /**
* called when closing TSP
*/
public void clear() {
storageGroupInfo.releaseStorageGroupMemCost(memCost);
- memCost = 0;
+ memCost = 0L;
+ metadataMemCost = 0L;
}
}