You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/11/01 05:35:03 UTC
[iotdb] branch master updated: update hot compaction config name
(#1859)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d1f6357 update hot compaction config name (#1859)
d1f6357 is described below
commit d1f63572a44bc8b5bc2d76af1b282981114db460
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Sun Nov 1 13:34:46 2020 +0800
update hot compaction config name (#1859)
---
.../resources/conf/iotdb-engine.properties | 8 ++--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 40 ++++++++--------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 24 +++++-----
.../level/LevelTsFileManagement.java | 53 ++++++++++++----------
.../iotdb/db/engine/upgrade/UpgradeTask.java | 2 +-
5 files changed, 65 insertions(+), 62 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 24d1808..6adf78f 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -265,19 +265,19 @@ tsfile_manage_strategy=NORMAL_STRATEGY
# Work when tsfile_manage_strategy is level_strategy.
# The max seq file num of each level. When file num exceeds this, the files in one level will merge to one.
-max_file_num_in_each_level=10
+seq_file_num_in_each_level=10
# Work when tsfile_manage_strategy is level_strategy.
# The max num of seq level.
-max_level_num=4
+seq_level_num=4
# Work when tsfile_manage_strategy is level_strategy.
# The max unseq file num of each level. When file num exceeds this, the files in one level will merge to one.
-max_unseq_file_num_in_each_level=10
+unseq_file_num_in_each_level=10
# Work when tsfile_manage_strategy is level_strategy.
# The max num of unseq level.
-max_unseq_level_num=2
+unseq_level_num=2
# Work when tsfile_manage_strategy is level_strategy.
# When merge point number reaches this, merge the files to the last level.
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 261a927..c6c0fb2 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -277,23 +277,23 @@ public class IoTDBConfig {
* Work when tsfile_manage_strategy is level_strategy. The max seq file num of each level. When
* file num exceeds this, the files in one level will merge to one.
*/
- private int maxFileNumInEachLevel = 10;
+ private int seqFileNumInEachLevel = 10;
/**
* Work when tsfile_manage_strategy is level_strategy. The max num of seq level.
*/
- private int maxLevelNum = 4;
+ private int seqLevelNum = 4;
/**
* Work when tsfile_manage_strategy is level_strategy. The max unseq file num of each level. When
* file num exceeds this, the files in one level will merge to one.
*/
- private int maxUnseqFileNumInEachLevel = 10;
+ private int unseqFileNumInEachLevel = 10;
/**
* Work when tsfile_manage_strategy is level_strategy. The max num of unseq level.
*/
- private int maxUnseqLevelNum = 2;
+ private int unseqLevelNum = 2;
/**
* whether to cache meta data(ChunkMetaData and TsFileMetaData) or not.
@@ -1376,36 +1376,36 @@ public class IoTDBConfig {
this.tsFileManagementStrategy = tsFileManagementStrategy;
}
- public int getMaxFileNumInEachLevel() {
- return maxFileNumInEachLevel;
+ public int getSeqFileNumInEachLevel() {
+ return seqFileNumInEachLevel;
}
- public void setMaxFileNumInEachLevel(int maxFileNumInEachLevel) {
- this.maxFileNumInEachLevel = maxFileNumInEachLevel;
+ public void setSeqFileNumInEachLevel(int seqFileNumInEachLevel) {
+ this.seqFileNumInEachLevel = seqFileNumInEachLevel;
}
- public int getMaxLevelNum() {
- return maxLevelNum;
+ public int getSeqLevelNum() {
+ return seqLevelNum;
}
- public void setMaxLevelNum(int maxLevelNum) {
- this.maxLevelNum = maxLevelNum;
+ public void setSeqLevelNum(int seqLevelNum) {
+ this.seqLevelNum = seqLevelNum;
}
- public int getMaxUnseqFileNumInEachLevel() {
- return maxUnseqFileNumInEachLevel;
+ public int getUnseqFileNumInEachLevel() {
+ return unseqFileNumInEachLevel;
}
- public void setMaxUnseqFileNumInEachLevel(int maxUnseqFileNumInEachLevel) {
- this.maxUnseqFileNumInEachLevel = maxUnseqFileNumInEachLevel;
+ public void setUnseqFileNumInEachLevel(int unseqFileNumInEachLevel) {
+ this.unseqFileNumInEachLevel = unseqFileNumInEachLevel;
}
- public int getMaxUnseqLevelNum() {
- return maxUnseqLevelNum;
+ public int getUnseqLevelNum() {
+ return unseqLevelNum;
}
- public void setMaxUnseqLevelNum(int maxUnseqLevelNum) {
- this.maxUnseqLevelNum = maxUnseqLevelNum;
+ public void setUnseqLevelNum(int unseqLevelNum) {
+ this.unseqLevelNum = unseqLevelNum;
}
public int getMergeChunkSubThreadNum() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 0463828..ef107b8 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -281,21 +281,21 @@ public class IoTDBDescriptor {
.getProperty("tsfile_manage_strategy",
conf.getTsFileManagementStrategy().toString())));
- conf.setMaxLevelNum(Integer.parseInt(properties
- .getProperty("max_level_num",
- Integer.toString(conf.getMaxLevelNum()))));
+ conf.setSeqLevelNum(Integer.parseInt(properties
+ .getProperty("seq_level_num",
+ Integer.toString(conf.getSeqLevelNum()))));
- conf.setMaxFileNumInEachLevel(Integer.parseInt(properties
- .getProperty("max_file_num_in_each_level",
- Integer.toString(conf.getMaxFileNumInEachLevel()))));
+ conf.setSeqFileNumInEachLevel(Integer.parseInt(properties
+ .getProperty("seq_file_num_in_each_level",
+ Integer.toString(conf.getSeqFileNumInEachLevel()))));
- conf.setMaxUnseqLevelNum(Integer.parseInt(properties
- .getProperty("max_unseq_level_num",
- Integer.toString(conf.getMaxUnseqLevelNum()))));
+ conf.setUnseqLevelNum(Integer.parseInt(properties
+ .getProperty("unseq_level_num",
+ Integer.toString(conf.getUnseqLevelNum()))));
- conf.setMaxUnseqFileNumInEachLevel(Integer.parseInt(properties
- .getProperty("max_unseq_file_num_in_each_level",
- Integer.toString(conf.getMaxUnseqFileNumInEachLevel()))));
+ conf.setUnseqFileNumInEachLevel(Integer.parseInt(properties
+ .getProperty("unseq_file_num_in_each_level",
+ Integer.toString(conf.getUnseqFileNumInEachLevel()))));
conf.setSyncEnable(Boolean
.parseBoolean(properties.getProperty("is_sync_enable",
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java
index 575e645..490ffe2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/tsfilemanagement/level/LevelTsFileManagement.java
@@ -65,13 +65,15 @@ import org.slf4j.LoggerFactory;
public class LevelTsFileManagement extends TsFileManagement {
private static final Logger logger = LoggerFactory.getLogger(LevelTsFileManagement.class);
- private final int maxLevelNum = IoTDBDescriptor.getInstance().getConfig().getMaxLevelNum();
- private final int maxFileNumInEachLevel = IoTDBDescriptor.getInstance().getConfig()
- .getMaxFileNumInEachLevel();
- private final int maxUnseqLevelNum = IoTDBDescriptor.getInstance().getConfig()
- .getMaxUnseqLevelNum();
- private final int maxUnseqFileNumInEachLevel = IoTDBDescriptor.getInstance().getConfig()
- .getMaxFileNumInEachLevel();
+
+ private final int seqLevelNum = IoTDBDescriptor.getInstance().getConfig().getSeqLevelNum();
+ private final int seqFileNumInEachLevel = IoTDBDescriptor.getInstance().getConfig()
+ .getSeqFileNumInEachLevel();
+ private final int unseqLevelNum = IoTDBDescriptor.getInstance().getConfig()
+ .getUnseqLevelNum();
+ private final int unseqFileNumInEachLevel = IoTDBDescriptor.getInstance().getConfig()
+ .getSeqFileNumInEachLevel();
+
private final boolean isForceFullMerge = IoTDBDescriptor.getInstance().getConfig()
.isForceFullMerge();
// First map is partition list; Second list is level list; Third list is file list in level;
@@ -90,10 +92,10 @@ public class LevelTsFileManagement extends TsFileManagement {
for (TsFileResource mergeTsFile : mergeTsFiles) {
deleteLevelFile(mergeTsFile);
}
- for (int i = 0; i < maxLevelNum; i++) {
+ for (int i = 0; i < seqLevelNum; i++) {
sequenceTsFileResources.get(timePartitionId).get(i).removeAll(mergeTsFiles);
}
- for (int i = 0; i < maxUnseqLevelNum; i++) {
+ for (int i = 0; i < unseqLevelNum; i++) {
unSequenceTsFileResources.get(timePartitionId).get(i).removeAll(mergeTsFiles);
}
}
@@ -117,11 +119,11 @@ public class LevelTsFileManagement extends TsFileManagement {
List<TsFileResource> result = new ArrayList<>();
if (sequence) {
for (List<TreeSet<TsFileResource>> sequenceTsFileList : sequenceTsFileResources.values()) {
- result.addAll(sequenceTsFileList.get(maxLevelNum - 1));
+ result.addAll(sequenceTsFileList.get(seqLevelNum - 1));
}
} else {
for (List<List<TsFileResource>> unSequenceTsFileList : unSequenceTsFileResources.values()) {
- result.addAll(unSequenceTsFileList.get(maxUnseqLevelNum - 1));
+ result.addAll(unSequenceTsFileList.get(unseqLevelNum - 1));
}
}
return result;
@@ -190,7 +192,7 @@ public class LevelTsFileManagement extends TsFileManagement {
long timePartitionId = tsFileResource.getTimePartition();
int level = getMergeLevel(tsFileResource.getTsFile());
if (sequence) {
- if (level <= maxLevelNum - 1) {
+ if (level <= seqLevelNum - 1) {
// current file has too high level
sequenceTsFileResources
.computeIfAbsent(timePartitionId, this::newSequenceTsFileResources).get(level)
@@ -198,11 +200,11 @@ public class LevelTsFileManagement extends TsFileManagement {
} else {
// current file has normal level
sequenceTsFileResources
- .computeIfAbsent(timePartitionId, this::newSequenceTsFileResources).get(maxLevelNum - 1)
+ .computeIfAbsent(timePartitionId, this::newSequenceTsFileResources).get(seqLevelNum - 1)
.add(tsFileResource);
}
} else {
- if (level <= maxUnseqLevelNum - 1) {
+ if (level <= unseqLevelNum - 1) {
// current file has too high level
unSequenceTsFileResources
.computeIfAbsent(timePartitionId, this::newUnSequenceTsFileResources).get(level)
@@ -211,7 +213,7 @@ public class LevelTsFileManagement extends TsFileManagement {
// current file has normal level
unSequenceTsFileResources
.computeIfAbsent(timePartitionId, this::newUnSequenceTsFileResources)
- .get(maxUnseqLevelNum - 1).add(tsFileResource);
+ .get(unseqLevelNum - 1).add(tsFileResource);
}
}
}
@@ -280,14 +282,14 @@ public class LevelTsFileManagement extends TsFileManagement {
if (sequence) {
for (List<TreeSet<TsFileResource>> partitionSequenceTsFileResource : sequenceTsFileResources
.values()) {
- for (int i = maxLevelNum - 1; i >= 0; i--) {
+ for (int i = seqLevelNum - 1; i >= 0; i--) {
result += partitionSequenceTsFileResource.get(i).size();
}
}
} else {
for (List<List<TsFileResource>> partitionUnSequenceTsFileResource : unSequenceTsFileResources
.values()) {
- for (int i = maxUnseqLevelNum - 1; i >= 0; i--) {
+ for (int i = unseqLevelNum - 1; i >= 0; i--) {
result += partitionUnSequenceTsFileResource.get(i).size();
}
}
@@ -395,12 +397,12 @@ public class LevelTsFileManagement extends TsFileManagement {
forkTsFileList(
forkedSequenceTsFileResources,
sequenceTsFileResources.computeIfAbsent(timePartition, this::newSequenceTsFileResources),
- maxLevelNum);
+ seqLevelNum);
forkTsFileList(
forkedUnSequenceTsFileResources,
unSequenceTsFileResources
.computeIfAbsent(timePartition, this::newUnSequenceTsFileResources),
- maxUnseqLevelNum);
+ unseqLevelNum);
}
private void forkTsFileList(
@@ -422,13 +424,14 @@ public class LevelTsFileManagement extends TsFileManagement {
@Override
protected void merge(long timePartition) {
- merge(forkedSequenceTsFileResources, true, timePartition, maxLevelNum, maxFileNumInEachLevel);
- if (maxUnseqLevelNum <= 1 && forkedUnSequenceTsFileResources.size() > 0) {
+ merge(forkedSequenceTsFileResources, true, timePartition, seqLevelNum,
+ seqFileNumInEachLevel);
+ if (unseqLevelNum <= 1 && forkedUnSequenceTsFileResources.size() > 0) {
merge(isForceFullMerge, getTsFileList(true), forkedUnSequenceTsFileResources.get(0),
Long.MAX_VALUE);
} else {
- merge(forkedUnSequenceTsFileResources, false, timePartition, maxUnseqLevelNum,
- maxUnseqFileNumInEachLevel);
+ merge(forkedUnSequenceTsFileResources, false, timePartition, unseqLevelNum,
+ unseqFileNumInEachLevel);
}
}
@@ -513,7 +516,7 @@ public class LevelTsFileManagement extends TsFileManagement {
private List<TreeSet<TsFileResource>> newSequenceTsFileResources(Long k) {
List<TreeSet<TsFileResource>> newSequenceTsFileResources = new CopyOnWriteArrayList<>();
- for (int i = 0; i < maxLevelNum; i++) {
+ for (int i = 0; i < seqLevelNum; i++) {
newSequenceTsFileResources.add(new TreeSet<>(
(o1, o2) -> {
int rangeCompare = Long
@@ -528,7 +531,7 @@ public class LevelTsFileManagement extends TsFileManagement {
private List<List<TsFileResource>> newUnSequenceTsFileResources(Long k) {
List<List<TsFileResource>> newUnSequenceTsFileResources = new CopyOnWriteArrayList<>();
- for (int i = 0; i < maxUnseqLevelNum; i++) {
+ for (int i = 0; i < unseqLevelNum; i++) {
newUnSequenceTsFileResources.add(new CopyOnWriteArrayList<>());
}
return newUnSequenceTsFileResources;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
index 7f4e4ca..015cc01 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java
@@ -43,7 +43,7 @@ public class UpgradeTask extends WrappedRunnable {
private TsFileResource upgradeResource;
private static final Logger logger = LoggerFactory.getLogger(UpgradeTask.class);
private static final String COMMA_SEPERATOR = ",";
- private static final int maxLevelNum = IoTDBDescriptor.getInstance().getConfig().getMaxLevelNum();
+ private static final int maxLevelNum = IoTDBDescriptor.getInstance().getConfig().getSeqLevelNum();
private FSFactory fsFactory = FSFactoryProducer.getFSFactory();