You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/05/14 03:46:42 UTC
[iotdb] branch rel/1.1 updated: build device timeIndex when validating files after compaction (#9832)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new ee2218a608b build device timeIndex when validating files after compaction (#9832)
ee2218a608b is described below
commit ee2218a608b9bb339e63d74c21d4c7d27c73fd2e
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Sun May 14 11:46:35 2023 +0800
build device timeIndex when validating files after compaction (#9832)
---
.../compaction/execute/utils/CompactionUtils.java | 22 +++++++++++-----------
.../db/engine/storagegroup/TsFileResource.java | 5 +++++
.../engine/compaction/AbstractCompactionTest.java | 3 +--
.../FastCrossCompactionPerformerTest.java | 5 +++++
...eCompactionWithFastPerformerValidationTest.java | 16 +++++++++++++---
5 files changed, 35 insertions(+), 16 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
index e60d4d74096..5e873bea69f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.timeindex.DeviceTimeIndex;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
@@ -255,25 +256,24 @@ public class CompactionUtils {
}
public static boolean validateTsFileResources(
- TsFileManager manager, String storageGroupName, long timePartition) {
+ TsFileManager manager, String storageGroupName, long timePartition) throws IOException {
List<TsFileResource> resources =
manager.getOrCreateSequenceListByTimePartition(timePartition).getArrayList();
- resources.sort(
- (f1, f2) ->
- Long.compareUnsigned(
- Long.parseLong(f1.getTsFile().getName().split("-")[0]),
- Long.parseLong(f2.getTsFile().getName().split("-")[0])));
+
// deviceID -> <TsFileResource, last end time>
Map<String, Pair<TsFileResource, Long>> lastEndTimeMap = new HashMap<>();
for (TsFileResource resource : resources) {
+ DeviceTimeIndex timeIndex;
if (resource.getTimeIndexType() != 1) {
- // if time index is not device time index, then skip it
- continue;
+ // if time index is not device time index, then deserialize it from resource file
+ timeIndex = resource.buildDeviceTimeIndex();
+ } else {
+ timeIndex = (DeviceTimeIndex) resource.getTimeIndex();
}
- Set<String> devices = resource.getDevices();
+ Set<String> devices = timeIndex.getDevices();
for (String device : devices) {
- long currentStartTime = resource.getStartTime(device);
- long currentEndTime = resource.getEndTime(device);
+ long currentStartTime = timeIndex.getStartTime(device);
+ long currentEndTime = timeIndex.getEndTime(device);
Pair<TsFileResource, Long> lastDeviceInfo =
lastEndTimeMap.computeIfAbsent(device, x -> new Pair<>(null, Long.MIN_VALUE));
long lastEndTime = lastDeviceInfo.right;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 340d2b3dfac..510b05e0015 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -435,6 +435,11 @@ public class TsFileResource {
}
}
+ /** Only used for compaction to validate tsfile. */
+ public ITimeIndex getTimeIndex() {
+ return timeIndex;
+ }
+
/**
* Whether this TsFileResource contains this device, if false, it must not contain this device, if
* true, it may or may not contain this device
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
index f4a7528173c..70ba4fa38c3 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
@@ -65,7 +65,6 @@ import org.junit.Assert;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -486,7 +485,7 @@ public class AbstractCompactionTest {
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
- Collections.emptyList(),
+ tsFileManager.getTsFileList(false),
true);
List<TimeValuePair> timeseriesData = entry.getValue();
tmpSourceDatas.put(entry.getKey(), new ArrayList<>(timeseriesData));
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java
index 0b9c3c95ece..fa236ea2da4 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/FastCrossCompactionPerformerTest.java
@@ -3924,6 +3924,11 @@ public class FastCrossCompactionPerformerTest extends AbstractCompactionTest {
Assert.assertEquals(0, FileReaderManager.getInstance().getClosedFileReaderMap().size());
Assert.assertEquals(0, FileReaderManager.getInstance().getUnclosedFileReaderMap().size());
CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG);
+ tsFileManager.addAll(targetResources, true);
+ targetResources.get(3).degradeTimeIndex();
+ targetResources.get(2).degradeTimeIndex();
+ Assert.assertTrue(
+ CompactionUtils.validateTsFileResources(tsFileManager, COMPACTION_TEST_SG, 0));
List<String> deviceIdList = new ArrayList<>();
deviceIdList.add(COMPACTION_TEST_SG + PATH_SEPARATOR + "d0");
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index 6b3be949db9..2f7c1bdad2c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -2194,6 +2194,17 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
tsFileManager.getTsFileList(true).get(i).degradeTimeIndex();
}
+ // meet overlap files
+ Assert.assertFalse(
+ CompactionUtils.validateTsFileResources(tsFileManager, COMPACTION_TEST_SG, 0));
+
+ tsFileManager.getTsFileList(true).get(0).deserialize();
+ tsFileManager.getTsFileList(true).get(1).deserialize();
+ tsFileManager.getTsFileList(true).get(0).degradeTimeIndex();
+ tsFileManager.getTsFileList(true).get(1).degradeTimeIndex();
+ Assert.assertTrue(
+ CompactionUtils.validateTsFileResources(tsFileManager, COMPACTION_TEST_SG, 0));
+
// seq file 4,5 and 6 are being compacted by inner space compaction
List<TsFileResource> sourceFiles = new ArrayList<>();
sourceFiles.add(seqResources.get(4));
@@ -2204,9 +2215,8 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
InnerSpaceCompactionTask innerSpaceCompactionTask =
new InnerSpaceCompactionTask(
0, tsFileManager, sourceFiles, true, performer, new AtomicInteger(0), 0);
- if (!CompactionUtils.validateTsFileResources(tsFileManager, COMPACTION_TEST_SG, 0)) {
- Assert.fail("meet overlap seq files");
- }
+ Assert.assertTrue(
+ CompactionUtils.validateTsFileResources(tsFileManager, COMPACTION_TEST_SG, 0));
innerSpaceCompactionTask.start();
validateSeqFiles(true);
}