You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/11/06 15:17:53 UTC
[iotdb] branch master updated: add enable unseq compaction (#1972)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c0d731d add enable unseq compaction (#1972)
c0d731d is described below
commit c0d731dbd5380295df5036504b7524a2ade50bbf
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Fri Nov 6 23:16:54 2020 +0800
add enable unseq compaction (#1972)
---
server/src/assembly/resources/conf/iotdb-engine.properties | 4 ++++
.../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 14 ++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 ++++++----
.../compaction/level/LevelCompactionTsFileManagement.java | 6 ++++--
4 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index ae71650..e72b908 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -277,6 +277,10 @@ default_fill_interval=-1
compaction_strategy=LEVEL_COMPACTION
# Works when the compaction_strategy is LEVEL_COMPACTION.
+# Whether to merge unseq files into seq files or not.
+enable_unseq_compaction = true
+
+# Works when the compaction_strategy is LEVEL_COMPACTION.
# The max seq file num of each level.
# When the num of files in one level exceeds this,
# the files in this level will merge to one and put to upper level.
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 54b4528..e993979 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -308,6 +308,12 @@ public class IoTDBConfig {
/**
* Works when the compaction_strategy is LEVEL_COMPACTION.
+ * Whether to merge unseq files into seq files or not.
+ */
+ private boolean enableUnseqCompaction = true;
+
+ /**
+ * Works when the compaction_strategy is LEVEL_COMPACTION.
* The max seq file num of each level.
* When the num of files in one level exceeds this,
* the files in this level will merge to one and put to upper level.
@@ -1442,6 +1448,14 @@ public class IoTDBConfig {
this.compactionStrategy = compactionStrategy;
}
+ public boolean isEnableUnseqCompaction() {
+ return enableUnseqCompaction;
+ }
+
+ public void setEnableUnseqCompaction(boolean enableUnseqCompaction) {
+ this.enableUnseqCompaction = enableUnseqCompaction;
+ }
+
public int getSeqFileNumInEachLevel() {
return seqFileNumInEachLevel;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index de14763..d7a7058 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -307,6 +307,10 @@ public class IoTDBDescriptor {
.getProperty("compaction_strategy",
conf.getCompactionStrategy().toString())));
+ conf.setEnableUnseqCompaction(Boolean.parseBoolean(
+ properties.getProperty("enable_unseq_compaction",
+ Boolean.toString(conf.isEnableUnseqCompaction()))));
+
conf.setSeqLevelNum(Integer.parseInt(properties
.getProperty("seq_level_num",
Integer.toString(conf.getSeqLevelNum()))));
@@ -770,7 +774,8 @@ public class IoTDBDescriptor {
}
private void initMemoryAllocate(Properties properties) {
- String memoryAllocateProportion = properties.getProperty("write_read_schema_free_memory_proportion");
+ String memoryAllocateProportion = properties
+ .getProperty("write_read_schema_free_memory_proportion");
if (memoryAllocateProportion != null) {
String[] proportions = memoryAllocateProportion.split(":");
int proportionSum = 0;
@@ -831,9 +836,6 @@ public class IoTDBDescriptor {
/**
* Get default encode algorithm by data type
- *
- * @param dataType
- * @return
*/
public TSEncoding getDefualtEncodingByType(TSDataType dataType) {
switch (dataType) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index cba29a5..2c47e3a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -68,6 +68,8 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
private final int unseqFileNumInEachLevel = IoTDBDescriptor.getInstance().getConfig()
.getSeqFileNumInEachLevel();
+ private final boolean enableUnseqCompaction = IoTDBDescriptor.getInstance().getConfig()
+ .isEnableUnseqCompaction();
private final boolean isForceFullMerge = IoTDBDescriptor.getInstance().getConfig()
.isForceFullMerge();
// First map is partition list; Second list is level list; Third list is file list in level;
@@ -424,7 +426,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
protected void merge(long timePartition) {
merge(forkedSequenceTsFileResources, true, timePartition, seqLevelNum,
seqFileNumInEachLevel);
- if (unseqLevelNum <= 1 && forkedUnSequenceTsFileResources.size() > 0) {
+ if (enableUnseqCompaction && unseqLevelNum <= 1 && forkedUnSequenceTsFileResources.size() > 0) {
merge(isForceFullMerge, getTsFileList(true), forkedUnSequenceTsFileResources.get(0),
Long.MAX_VALUE);
} else {
@@ -444,7 +446,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
for (int i = 0; i < currMaxLevel - 1; i++) {
if (currMaxFileNumInEachLevel <= mergeResources.get(i).size()) {
//level is numbered from 0
- if (!sequence && i == currMaxLevel - 2) {
+ if (enableUnseqCompaction && !sequence && i == currMaxLevel - 2) {
// do not merge current unseq file level to upper level and just merge all of them to seq file
merge(isForceFullMerge, getTsFileList(true), mergeResources.get(i), Long.MAX_VALUE);
} else {