You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/01/13 14:12:12 UTC
[iotdb] branch master updated: [IOTDB-2385] Fix write degrade after limit chunk size in inner space compaction (#4798)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7fea9c8 [IOTDB-2385] Fix write degrade after limit chunk size in inner space compaction (#4798)
7fea9c8 is described below
commit 7fea9c82b40f8ca8bdcec7be0434eef42ad7cb39
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Thu Jan 13 22:11:36 2022 +0800
[IOTDB-2385] Fix write degrade after limit chunk size in inner space compaction (#4798)
---
.../resources/conf/iotdb-engine.properties | 4 +--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +--
.../inner/utils/InnerSpaceCompactionUtils.java | 1 +
.../inner/utils/MultiTsFileDeviceIterator.java | 31 ++++++++++------------
.../db/engine/storagegroup/TsFileResource.java | 8 +++---
.../encoding/encoder/DeltaBinaryEncoder.java | 10 +++----
.../iotdb/tsfile/read/TsFileSequenceReader.java | 5 ++++
7 files changed, 34 insertions(+), 29 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 179f2f7..dfe475e 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -521,11 +521,11 @@ timestamp_precision=ms
# The interval of compaction task schedule
# Datatype: long, Unit: ms
-# compaction_schedule_interval=10000
+# compaction_schedule_interval=60000
# The interval of compaction task submission
# Datatype: long, Unit: ms
-# compaction_submission_interval=1000
+# compaction_submission_interval=60000
# The limit of write throughput merge can reach per second
# Datatype: int
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 66d82b7..a11cfe1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -430,10 +430,10 @@ public class IoTDBConfig {
private int mergePagePointNumberThreshold = 100;
/** The interval of compaction task schedulation in each virtual storage group. The unit is ms. */
- private long compactionScheduleInterval = 10_000L;
+ private long compactionScheduleInterval = 60_000L;
/** The interval of compaction task submission from queue in CompactionTaskMananger */
- private long compactionSubmissionInterval = 1_000L;
+ private long compactionSubmissionInterval = 60_000L;
/**
* The max open file num in each unseq compaction task. We use the unseq file num as the open file
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java
index 2a94b98..ce1cba6 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java
@@ -78,6 +78,7 @@ public class InnerSpaceCompactionUtils {
} else {
compactNotAlignedSeries(device, targetResource, writer, deviceIterator, sequence);
}
+
writer.endChunkGroup();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/MultiTsFileDeviceIterator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/MultiTsFileDeviceIterator.java
index 12963c1..2725294 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/MultiTsFileDeviceIterator.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/MultiTsFileDeviceIterator.java
@@ -44,15 +44,14 @@ import java.util.stream.Collectors;
public class MultiTsFileDeviceIterator implements AutoCloseable {
private List<TsFileResource> tsFileResources;
- private Map<TsFileResource, TsFileSequenceReader> readerMap =
- new TreeMap<>((o1, o2) -> TsFileResource.compareFileName(o1.getTsFile(), o2.getTsFile()));
- private Map<TsFileResource, TsFileDeviceIterator> deviceIteratorMap =
- new TreeMap<>((o1, o2) -> TsFileResource.compareFileName(o1.getTsFile(), o2.getTsFile()));
+ private Map<TsFileResource, TsFileSequenceReader> readerMap = new HashMap<>();
+ private Map<TsFileResource, TsFileDeviceIterator> deviceIteratorMap = new HashMap<>();
private Map<TsFileResource, List<Modification>> modificationCache = new HashMap<>();
private Pair<String, Boolean> currentDevice = null;
public MultiTsFileDeviceIterator(List<TsFileResource> tsFileResources) throws IOException {
this.tsFileResources = new ArrayList<>(tsFileResources);
+ Collections.sort(this.tsFileResources, TsFileResource::compareFileName);
try {
for (TsFileResource tsFileResource : this.tsFileResources) {
TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFilePath());
@@ -210,20 +209,20 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
private LinkedList<String> seriesInThisIteration = new LinkedList<>();
// tsfile sequence reader -> series -> list<ChunkMetadata>
private Map<TsFileSequenceReader, Map<String, List<ChunkMetadata>>> chunkMetadataCacheMap =
- new TreeMap<>(new InnerSpaceCompactionUtils.TsFileNameComparator());
+ new HashMap<>();
// this map cache the chunk metadata list iterator for each tsfile
// the iterator return a batch of series and all chunk metadata of these series in this tsfile
- private Map<TsFileSequenceReader, Iterator<Map<String, List<ChunkMetadata>>>>
- chunkMetadataIteratorMap =
- new TreeMap<>(new InnerSpaceCompactionUtils.TsFileNameComparator());
+ private Map<TsFileResource, Iterator<Map<String, List<ChunkMetadata>>>>
+ chunkMetadataIteratorMap = new HashMap<>();
private MeasurementIterator(Map<TsFileResource, TsFileSequenceReader> readerMap, String device)
throws IOException {
this.readerMap = readerMap;
this.device = device;
- for (TsFileSequenceReader reader : readerMap.values()) {
+ for (TsFileResource resource : tsFileResources) {
+ TsFileSequenceReader reader = readerMap.get(resource);
chunkMetadataIteratorMap.put(
- reader, reader.getMeasurementChunkMetadataListMapIterator(device));
+ resource, reader.getMeasurementChunkMetadataListMapIterator(device));
chunkMetadataCacheMap.put(reader, new TreeMap<>());
}
}
@@ -243,14 +242,12 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
private boolean collectSeries() {
String lastSeries = null;
List<String> tempCollectedSeries = new ArrayList<>();
- for (Map.Entry<TsFileSequenceReader, Map<String, List<ChunkMetadata>>>
- chunkMetadataListCacheForMergeEntry : chunkMetadataCacheMap.entrySet()) {
- TsFileSequenceReader reader = chunkMetadataListCacheForMergeEntry.getKey();
- Map<String, List<ChunkMetadata>> chunkMetadataListMap =
- chunkMetadataListCacheForMergeEntry.getValue();
+ for (TsFileResource resource : tsFileResources) {
+ TsFileSequenceReader reader = readerMap.get(resource);
+ Map<String, List<ChunkMetadata>> chunkMetadataListMap = chunkMetadataCacheMap.get(reader);
if (chunkMetadataListMap.size() == 0) {
- if (chunkMetadataIteratorMap.get(reader).hasNext()) {
- chunkMetadataListMap = chunkMetadataIteratorMap.get(reader).next();
+ if (chunkMetadataIteratorMap.get(resource).hasNext()) {
+ chunkMetadataListMap = chunkMetadataIteratorMap.get(resource).next();
chunkMetadataCacheMap.put(reader, chunkMetadataListMap);
} else {
continue;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index f9ab75d..02655d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -872,9 +872,11 @@ public class TsFileResource {
}
// ({systemTime}-{versionNum}-{innerMergeNum}-{crossMergeNum}.tsfile)
- public static int compareFileName(File o1, File o2) {
- String[] items1 = o1.getName().replace(TSFILE_SUFFIX, "").split(FILE_NAME_SEPARATOR);
- String[] items2 = o2.getName().replace(TSFILE_SUFFIX, "").split(FILE_NAME_SEPARATOR);
+ public static int compareFileName(TsFileResource o1, TsFileResource o2) {
+ String[] items1 =
+ o1.getTsFile().getName().replace(TSFILE_SUFFIX, "").split(FILE_NAME_SEPARATOR);
+ String[] items2 =
+ o2.getTsFile().getName().replace(TSFILE_SUFFIX, "").split(FILE_NAME_SEPARATOR);
long ver1 = Long.parseLong(items1[0]);
long ver2 = Long.parseLong(items2[0]);
int cmp = Long.compare(ver1, ver2);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index a72c44b..b958460 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -150,8 +150,8 @@ public abstract class DeltaBinaryEncoder extends Encoder {
return width;
}
- private void calcDelta(Integer value) {
- Integer delta = value - previousValue; // calculate delta
+ private void calcDelta(int value) {
+ int delta = value - previousValue; // calculate delta
if (delta < minDeltaBase) {
minDeltaBase = delta;
}
@@ -249,8 +249,8 @@ public abstract class DeltaBinaryEncoder extends Encoder {
reset();
}
- private void calcDelta(Long value) {
- Long delta = value - previousValue; // calculate delta
+ private void calcDelta(long value) {
+ long delta = value - previousValue; // calculate delta
if (delta < minDeltaBase) {
minDeltaBase = delta;
}
@@ -268,7 +268,7 @@ public abstract class DeltaBinaryEncoder extends Encoder {
}
}
- private int getValueWidth(Long v) {
+ private int getValueWidth(long v) {
return 64 - Long.numberOfLeadingZeros(v);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 52c830d..d99b3ba 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1734,4 +1734,9 @@ public class TsFileSequenceReader implements AutoCloseable {
throw e;
}
}
+
+ @Override
+ public int hashCode() {
+ return file.hashCode();
+ }
}