You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/11/30 10:41:21 UTC
[iotdb] branch IOTDB-5092 created (now e512a9a7c0)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a change to branch IOTDB-5092
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at e512a9a7c0 set cross_compaction to true and add compaction validation
This branch includes the following new commits:
new e512a9a7c0 set cross_compaction to true and add compaction validation
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: set cross_compaction to true and add compaction validation
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch IOTDB-5092
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e512a9a7c062eb30388c1316cb2540f6f2e7cfa8
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Wed Nov 30 18:41:06 2022 +0800
set cross_compaction to true and add compaction validation
---
docs/UserGuide/Reference/Common-Config-Manual.md | 10 ++++++
.../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +++++
.../resources/conf/iotdb-common.properties | 7 ++--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++
.../db/engine/compaction/CompactionUtils.java | 40 ++++++++++++++++++++++
.../compaction/cross/CrossSpaceCompactionTask.java | 12 +++++++
.../compaction/inner/InnerSpaceCompactionTask.java | 11 ++++++
8 files changed, 103 insertions(+), 2 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md
index 8356e196c6..504824ccdc 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -1008,6 +1008,16 @@ Different configuration parameters take effect in the following three ways:
|Default| 4 |
|Effective| After restart system |
+* enable\_compaction\_validation
+
+|Name| enable\_compaction\_validation |
+|:---:|:----------------------------------------------------------------|
+|Description| Enable the check of sequence tsfile time range after compaction |
+|Type| Boolean |
+|Default| true |
+|Effective| After restart system |
+
+
### Write Ahead Log Configuration
* wal\_mode
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index 0066cb1e43..3fa5556310 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -1063,6 +1063,15 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
|默认值| 4 |
|改后生效方式| 重启服务生效|
+* enable\_compaction\_validation
+
+|名字| enable\_compaction\_validation |
+|:---:|:--|
+|描述| 开启合并结束后对顺序文件时间范围的检查 |
+|类型| Boolean |
+|默认值| true |
+|改后生效方式| 重启服务生效|
+
### 写前日志配置
* wal\_mode
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index deb66da80d..930ab46ba2 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -532,9 +532,8 @@
# enable_unseq_space_compaction=true
# cross space compaction: compact the unsequence files into the overlapped sequence files
-# It is recommended to disable this in version 1.0.0, and can enable it in version 1.0.1
# Datatype: boolean
-enable_cross_space_compaction=false
+# enable_cross_space_compaction=true
# the selector of cross space compaction task
# Options: rewrite
@@ -630,6 +629,10 @@ enable_cross_space_compaction=false
# Datatype: int
# sub_compaction_thread_count=4
+# Enable the check of sequence tsfile time range after compaction
+# Datatype: boolean
+# enable_compaction_validation=true
+
####################
### Write Ahead Log Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 6a1f46f760..0f923aef2e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -487,6 +487,8 @@ public class IoTDBConfig {
*/
private int subCompactionTaskNum = 4;
+ private boolean enableCompactionValidation = true;
+
/** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
private boolean metaDataCacheEnable = true;
@@ -3577,4 +3579,12 @@ public class IoTDBConfig {
public void setSchemaRatisLogMaxMB(long schemaRatisLogMaxMB) {
this.schemaRatisLogMaxMB = schemaRatisLogMaxMB;
}
+
+ public boolean isEnableCompactionValidation() {
+ return enableCompactionValidation;
+ }
+
+ public void setEnableCompactionValidation(boolean enableCompactionValidation) {
+ this.enableCompactionValidation = enableCompactionValidation;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 5373b4d7b0..bba0dd92c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -673,6 +673,12 @@ public class IoTDBDescriptor {
"compaction_write_throughput_mb_per_sec",
Integer.toString(conf.getCompactionWriteThroughputMbPerSec()))));
+ conf.setEnableCompactionValidation(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_compaction_validation",
+ Boolean.toString(conf.isEnableCompactionValidation()))));
+
conf.setEnablePartialInsert(
Boolean.parseBoolean(
properties.getProperty(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java
index bb114710f5..d4b5a33e60 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.compaction;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -35,8 +36,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -256,4 +259,41 @@ public class CompactionUtils {
}
}
}
+
+ public static boolean validateTsFileResources(
+ TsFileManager manager, String storageGroupName, long timePartition) {
+ List<TsFileResource> resources =
+ manager.getSequenceListByTimePartition(timePartition).getArrayList();
+ resources.sort(
+ (f1, f2) ->
+ Long.compareUnsigned(
+ Long.parseLong(f1.getTsFile().getName().split("-")[0]),
+ Long.parseLong(f2.getTsFile().getName().split("-")[0])));
+ Map<String, Long> lastEndTimeMap = new HashMap<>();
+ TsFileResource prevTsFileResource = null;
+ for (TsFileResource resource : resources) {
+ Set<String> devices = resource.getDevices();
+ for (String device : devices) {
+ long currentStartTime = resource.getStartTime(device);
+ long currentEndTime = resource.getEndTime(device);
+ long lastEndTime = lastEndTimeMap.computeIfAbsent(device, x -> Long.MIN_VALUE);
+ if (lastEndTime >= currentStartTime) {
+ logger.error(
+ "{} Device {} is overlapped between {} and {}, end time in {} is {}, start time in {} is {}",
+ storageGroupName,
+ device,
+ prevTsFileResource,
+ resource,
+ prevTsFileResource,
+ lastEndTime,
+ resource,
+ currentStartTime);
+ return false;
+ }
+ lastEndTimeMap.put(device, currentEndTime);
+ }
+ prevTsFileResource = resource;
+ }
+ return true;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
index 960073aea3..4a555525d3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.cross;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.TsFileMetricManager;
import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler;
import org.apache.iotdb.db.engine.compaction.CompactionUtils;
@@ -169,6 +170,17 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
timePartition,
true);
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation()
+ && !CompactionUtils.validateTsFileResources(
+ tsFileManager, storageGroupName, timePartition)) {
+ LOGGER.error(
+ "Failed to pass compaction validation, source sequence files is: {}, unsequence files is {}, target files is {}",
+ selectedSequenceFiles,
+ selectedUnsequenceFiles,
+ targetTsfileResourceList);
+ throw new RuntimeException("Failed to pass compaction validation");
+ }
+
releaseReadAndLockWrite(selectedSequenceFiles);
releaseReadAndLockWrite(selectedUnsequenceFiles);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
index 51d8554940..ca78d5d694 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.inner;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.TsFileMetricManager;
import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler;
import org.apache.iotdb.db.engine.compaction.CompactionUtils;
@@ -177,6 +178,16 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
false);
}
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation()
+ && !CompactionUtils.validateTsFileResources(
+ tsFileManager, storageGroupName, timePartition)) {
+ LOGGER.error(
+ "Failed to pass compaction validation, source files is: {}, target files is {}",
+ selectedTsFileResourceList,
+ targetTsFileList);
+ throw new RuntimeException("Failed to pass compaction validation");
+ }
+
LOGGER.info(
"{}-{} [Compaction] Compacted target files, try to get the write lock of source files",
storageGroupName,