You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/11/30 16:12:19 UTC
[iotdb] branch rel/1.0 updated: set cross_compaction to true and add compaction validation (#8268)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.0 by this push:
new 4c77331807 set cross_compaction to true and add compaction validation (#8268)
4c77331807 is described below
commit 4c77331807a71dbc351eb363b2144f9250718480
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Thu Dec 1 00:12:12 2022 +0800
set cross_compaction to true and add compaction validation (#8268)
---
docs/UserGuide/Reference/Common-Config-Manual.md | 10 ++++++
.../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +++++
.../resources/conf/iotdb-common.properties | 7 ++--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++
.../db/engine/compaction/CompactionUtils.java | 40 ++++++++++++++++++++++
.../compaction/cross/CrossSpaceCompactionTask.java | 12 +++++++
.../compaction/inner/InnerSpaceCompactionTask.java | 11 ++++++
8 files changed, 103 insertions(+), 2 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md
index 8356e196c6..504824ccdc 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -1008,6 +1008,16 @@ Different configuration parameters take effect in the following three ways:
|Default| 4 |
|Effective| After restart system |
+* enable\_compaction\_validation
+
+|Name| enable\_compaction\_validation |
+|:---:|:----------------------------------------------------------------|
+|Description| Enable the check of sequence tsfile time range after compaction |
+|Type| Boolean |
+|Default| true |
+|Effective| After restart system |
+
+
### Write Ahead Log Configuration
* wal\_mode
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index 0066cb1e43..3fa5556310 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -1063,6 +1063,15 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
|默认值| 4 |
|改后生效方式| 重启服务生效|
+* enable\_compaction\_validation
+
+|名字| enable\_compaction\_validation |
+|:---:|:--|
+|描述| 开启合并结束后对顺序文件时间范围的检查 |
+|类型| Boolean |
+|默认值| true |
+|改后生效方式| 重启服务生效|
+
### 写前日志配置
* wal\_mode
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index deb66da80d..930ab46ba2 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -532,9 +532,8 @@
# enable_unseq_space_compaction=true
# cross space compaction: compact the unsequence files into the overlapped sequence files
-# It is recommended to disable this in version 1.0.0, and can enable it in version 1.0.1
# Datatype: boolean
-enable_cross_space_compaction=false
+# enable_cross_space_compaction=true
# the selector of cross space compaction task
# Options: rewrite
@@ -630,6 +629,10 @@ enable_cross_space_compaction=false
# Datatype: int
# sub_compaction_thread_count=4
+# Enable the check of sequence tsfile time range after compaction
+# Datatype: boolean
+# enable_compaction_validation=true
+
####################
### Write Ahead Log Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 6a1f46f760..0f923aef2e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -487,6 +487,8 @@ public class IoTDBConfig {
*/
private int subCompactionTaskNum = 4;
+ private boolean enableCompactionValidation = true;
+
/** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
private boolean metaDataCacheEnable = true;
@@ -3577,4 +3579,12 @@ public class IoTDBConfig {
public void setSchemaRatisLogMaxMB(long schemaRatisLogMaxMB) {
this.schemaRatisLogMaxMB = schemaRatisLogMaxMB;
}
+
+ public boolean isEnableCompactionValidation() {
+ return enableCompactionValidation;
+ }
+
+ public void setEnableCompactionValidation(boolean enableCompactionValidation) {
+ this.enableCompactionValidation = enableCompactionValidation;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 5373b4d7b0..bba0dd92c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -673,6 +673,12 @@ public class IoTDBDescriptor {
"compaction_write_throughput_mb_per_sec",
Integer.toString(conf.getCompactionWriteThroughputMbPerSec()))));
+ conf.setEnableCompactionValidation(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_compaction_validation",
+ Boolean.toString(conf.isEnableCompactionValidation()))));
+
conf.setEnablePartialInsert(
Boolean.parseBoolean(
properties.getProperty(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java
index bb114710f5..d4b5a33e60 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.compaction;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -35,8 +36,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -256,4 +259,41 @@ public class CompactionUtils {
}
}
}
+
+ public static boolean validateTsFileResources(
+ TsFileManager manager, String storageGroupName, long timePartition) {
+ List<TsFileResource> resources =
+ manager.getSequenceListByTimePartition(timePartition).getArrayList();
+ resources.sort(
+ (f1, f2) ->
+ Long.compareUnsigned(
+ Long.parseLong(f1.getTsFile().getName().split("-")[0]),
+ Long.parseLong(f2.getTsFile().getName().split("-")[0])));
+ Map<String, Long> lastEndTimeMap = new HashMap<>();
+ TsFileResource prevTsFileResource = null;
+ for (TsFileResource resource : resources) {
+ Set<String> devices = resource.getDevices();
+ for (String device : devices) {
+ long currentStartTime = resource.getStartTime(device);
+ long currentEndTime = resource.getEndTime(device);
+ long lastEndTime = lastEndTimeMap.computeIfAbsent(device, x -> Long.MIN_VALUE);
+ if (lastEndTime >= currentStartTime) {
+ logger.error(
+ "{} Device {} is overlapped between {} and {}, end time in {} is {}, start time in {} is {}",
+ storageGroupName,
+ device,
+ prevTsFileResource,
+ resource,
+ prevTsFileResource,
+ lastEndTime,
+ resource,
+ currentStartTime);
+ return false;
+ }
+ lastEndTimeMap.put(device, currentEndTime);
+ }
+ prevTsFileResource = resource;
+ }
+ return true;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
index 960073aea3..4a555525d3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.cross;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.TsFileMetricManager;
import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler;
import org.apache.iotdb.db.engine.compaction.CompactionUtils;
@@ -169,6 +170,17 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
timePartition,
true);
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation()
+ && !CompactionUtils.validateTsFileResources(
+ tsFileManager, storageGroupName, timePartition)) {
+ LOGGER.error(
+ "Failed to pass compaction validation, source sequence files is: {}, unsequence files is {}, target files is {}",
+ selectedSequenceFiles,
+ selectedUnsequenceFiles,
+ targetTsfileResourceList);
+ throw new RuntimeException("Failed to pass compaction validation");
+ }
+
releaseReadAndLockWrite(selectedSequenceFiles);
releaseReadAndLockWrite(selectedUnsequenceFiles);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
index 51d8554940..ca78d5d694 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.inner;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.TsFileMetricManager;
import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler;
import org.apache.iotdb.db.engine.compaction.CompactionUtils;
@@ -177,6 +178,16 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
false);
}
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation()
+ && !CompactionUtils.validateTsFileResources(
+ tsFileManager, storageGroupName, timePartition)) {
+ LOGGER.error(
+ "Failed to pass compaction validation, source files is: {}, target files is {}",
+ selectedTsFileResourceList,
+ targetTsFileList);
+ throw new RuntimeException("Failed to pass compaction validation");
+ }
+
LOGGER.info(
"{}-{} [Compaction] Compacted target files, try to get the write lock of source files",
storageGroupName,