You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2023/02/03 15:50:35 UTC
[iotdb] 15/28: update modifyChunkMetadata optimize the double-loop for chunkmetadata and deletions
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 84bd6f1e356cb92922f359873fe3cc4b207a12c5
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jan 27 23:27:31 2023 +0800
update modifyChunkMetadata optimize the double-loop for chunkmetadata and deletions
---
.../resources/conf/iotdb-engine.properties | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 8 ++++----
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 24 ++++++++++++++--------
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 7 ++++++-
4 files changed, 26 insertions(+), 15 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 209e4fc389..4a9b6799ad 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -20,7 +20,7 @@
####################
### enable CPV
####################
-enable_CPV=false
+enable_CPV=true
####################
### Web Page Configuration
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index e9d333f0e8..f45461a069 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -266,10 +266,10 @@ public class IoTDBConfig {
private String indexRootFolder = "data" + File.separator + "index";
/** When a unSequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */
- private long unSeqTsFileSize = 1L;
+ private long unSeqTsFileSize = 1073741824;
/** When a sequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */
- private long seqTsFileSize = 1L;
+ private long seqTsFileSize = 1073741824;
/** When a memTable's size (in byte) exceeds this, the memtable is flushed to disk. */
private long memtableSizeThreshold = 1024 * 1024 * 1024L;
@@ -328,7 +328,7 @@ public class IoTDBConfig {
private int mergePagePointNumberThreshold = 100;
/** LEVEL_COMPACTION, NO_COMPACTION */
- private CompactionStrategy compactionStrategy = CompactionStrategy.LEVEL_COMPACTION;
+ private CompactionStrategy compactionStrategy = CompactionStrategy.NO_COMPACTION;
/**
* Works when the compaction_strategy is LEVEL_COMPACTION. Whether to merge unseq files into seq
@@ -431,7 +431,7 @@ public class IoTDBConfig {
/** Is performance tracing enable. */
private boolean enablePerformanceTracing = false;
- private boolean enableCPV = false;
+ private boolean enableCPV = true;
/** The display of stat performance interval in ms. */
private long performanceStatDisplayInterval = 60000;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index 8da3273068..9863716ba0 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -54,18 +54,24 @@ public class QueryUtils {
List<Deletion> deletions =
modifications.stream().map(e -> (Deletion) e).sorted().collect(Collectors.toList());
+ // sort chunkMetadatas by startTime
+ List<ChunkMetadata> sortedChunkMetadata =
+ chunkMetaData.stream().map(e -> (ChunkMetadata) e).sorted().collect(Collectors.toList());
+
+ // TODO add startPos for chunk to iterate deletes to speed up
+ int deleteStartIdx = 0;
// 对于每个chunkMetadata，先过滤掉时间范围上不重叠的删除操作，然后再根据版本高低判断是否应用mod
- for (int metaIndex = 0; metaIndex < chunkMetaData.size(); metaIndex++) {
- ChunkMetadata metaData = chunkMetaData.get(metaIndex);
- long startTime = metaData.getStartTime();
- long endTime = metaData.getEndTime();
- for (Deletion deletion : deletions) {
- long deleteStartTime = deletion.getStartTime();
- long deleteEndTime = deletion.getEndTime();
- if (deleteStartTime > endTime) {
+ for (int metaIndex = 0; metaIndex < sortedChunkMetadata.size(); metaIndex++) {
+ ChunkMetadata metaData = sortedChunkMetadata.get(metaIndex);
+ // for (Deletion deletion : deletions) {
+ for (int j = deleteStartIdx; j < deletions.size(); j++) {
+ // TODO use deleteStartIdx to avoid iterate from the first delete
+ Deletion deletion = deletions.get(j);
+ if (deletion.getStartTime() > metaData.getEndTime()) {
break;
}
- if (startTime > deleteEndTime) {
+ if (metaData.getStartTime() > deletion.getEndTime()) {
+ deleteStartIdx = j + 1; // TODO update startPos for chunk to iterate deletes to speed up
continue;
}
// then deals with deletes that overlap in time with the chunk
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 4ce0d92e9b..60494e223a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -34,7 +34,7 @@ import java.util.List;
import java.util.Objects;
/** Metadata of one chunk. */
-public class ChunkMetadata {
+public class ChunkMetadata implements Comparable<ChunkMetadata> {
private String measurementUid;
@@ -117,6 +117,11 @@ public class ChunkMetadata {
// this.version = chunkMetadata.version;
// }
+ @Override
+ public int compareTo(ChunkMetadata chunkMetadata) {
+ return Long.compare(statistics.getStartTime(), chunkMetadata.getStartTime());
+ }
+
@Override
public String toString() {
return String.format(