You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/11/15 15:24:55 UTC

(iotdb) branch master updated: Fix aligned series modification not matched in MultiTsFileDeviceIterator (#11544)

This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new de96005b9bd Fix aligned series modification not matched in MultiTsFileDeviceIterator (#11544)
de96005b9bd is described below

commit de96005b9bd7e0b61b303da1443c0f0889ccefaf
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Wed Nov 15 23:24:48 2023 +0800

    Fix aligned series modification not matched in MultiTsFileDeviceIterator (#11544)
---
 .../impl/ReadChunkCompactionPerformer.java         |   3 +-
 .../execute/utils/MultiTsFileDeviceIterator.java   |  16 ++-
 .../compaction/ReadChunkInnerCompactionTest.java   | 110 +++++++++++++++++++++
 3 files changed, 123 insertions(+), 6 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
index 80fc8b1dcb3..67fe8877918 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl;
 
+import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.exception.MetadataException;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -119,7 +120,7 @@ public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer {
       TsFileResource targetResource,
       CompactionTsFileWriter writer,
       MultiTsFileDeviceIterator deviceIterator)
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, IllegalPathException {
     checkThreadInterrupted();
     LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>> readerAndChunkMetadataList =
         deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 09fa3a85bcd..98aa372b30c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -339,7 +339,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
    */
   @SuppressWarnings({"squid:S1319", "squid:S135"})
   public LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>>
-      getReaderAndChunkMetadataForCurrentAlignedSeries() throws IOException {
+      getReaderAndChunkMetadataForCurrentAlignedSeries() throws IOException, IllegalPathException {
     if (currentDevice == null || !currentDevice.right) {
       return new LinkedList<>();
     }
@@ -374,7 +374,8 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
    * @param alignedChunkMetadataList list of aligned chunk metadata
    */
   private void applyModificationForAlignedChunkMetadataList(
-      TsFileResource tsFileResource, List<AlignedChunkMetadata> alignedChunkMetadataList) {
+      TsFileResource tsFileResource, List<AlignedChunkMetadata> alignedChunkMetadataList)
+      throws IllegalPathException {
     ModificationFile modificationFile = ModificationFile.getNormalMods(tsFileResource);
     if (!modificationFile.exists()) {
       return;
@@ -387,16 +388,21 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
     AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0);
     List<IChunkMetadata> valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList();
     List<List<Modification>> modificationForCurDevice = new ArrayList<>();
+    List<PartialPath> valueSeriesPaths = new ArrayList<>(valueChunkMetadataList.size());
     for (int i = 0; i < valueChunkMetadataList.size(); ++i) {
       modificationForCurDevice.add(new ArrayList<>());
+      IChunkMetadata valueChunkMetadata = valueChunkMetadataList.get(i);
+      valueSeriesPaths.add(
+          valueChunkMetadata == null
+              ? null
+              : new PartialPath(currentDevice.left, valueChunkMetadata.getMeasurementUid()));
     }
 
     for (Modification modification : modifications) {
       if (modification.getDevice().equals(currentDevice.left)) {
         for (int i = 0; i < valueChunkMetadataList.size(); ++i) {
-          IChunkMetadata chunkMetadata = valueChunkMetadataList.get(i);
-          if (chunkMetadata != null
-              && modification.getMeasurement().equals(chunkMetadata.getMeasurementUid())) {
+          PartialPath path = valueSeriesPaths.get(i);
+          if (path != null && modification.getPath().matchFullPath(path)) {
             modificationForCurDevice.get(i).add(modification);
           }
         }
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
index a3ca978bb3c..5e7920ed2b3 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iotdb.db.storageengine.dataregion.compaction;
 
+import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.exception.MetadataException;
 import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.path.PartialPath;
@@ -26,6 +27,7 @@ import org.apache.iotdb.db.exception.StorageEngineException;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
 import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
 import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
@@ -47,6 +49,7 @@ import org.junit.Test;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -416,6 +419,113 @@ public class ReadChunkInnerCompactionTest extends AbstractCompactionTest {
     Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
   }
 
+  @Test
+  public void testReadChunkPerformerWithEmptyTargetFile3()
+      throws IOException, IllegalPathException {
+    TsFileResource seqFile1 = createEmptyFileAndResource(true);
+    try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqFile1)) {
+      writer.startChunkGroup("d1");
+      writer.generateSimpleAlignedSeriesToCurrentDevice(
+          Arrays.asList("s1"),
+          new TimeRange[] {new TimeRange(1, 2)},
+          TSEncoding.PLAIN,
+          CompressionType.LZ4);
+      writer.endChunkGroup();
+      writer.endFile();
+    }
+    seqFile1
+        .getModFile()
+        .write(new Deletion(new PartialPath("root.testsg.d1.**"), Long.MAX_VALUE, Long.MAX_VALUE));
+    seqFile1.getModFile().close();
+    TsFileResource seqFile2 = createEmptyFileAndResource(true);
+    try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqFile2)) {
+      writer.startChunkGroup("d1");
+      writer.generateSimpleAlignedSeriesToCurrentDevice(
+          Arrays.asList("s1"),
+          new TimeRange[] {new TimeRange(5, 6)},
+          TSEncoding.PLAIN,
+          CompressionType.LZ4);
+      writer.endChunkGroup();
+      writer.endFile();
+    }
+    seqFile2
+        .getModFile()
+        .write(new Deletion(new PartialPath("root.testsg.d1.**"), Long.MAX_VALUE, Long.MAX_VALUE));
+    seqFile2.getModFile().close();
+    TsFileResource seqFile3 = createEmptyFileAndResource(true);
+    try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqFile3)) {
+      writer.startChunkGroup("d1");
+      writer.generateSimpleAlignedSeriesToCurrentDevice(
+          Arrays.asList("s1"),
+          new TimeRange[] {new TimeRange(10, 20)},
+          TSEncoding.PLAIN,
+          CompressionType.LZ4);
+      writer.endChunkGroup();
+      writer.endFile();
+    }
+    seqFile3
+        .getModFile()
+        .write(new Deletion(new PartialPath("root.testsg.d1.**"), Long.MAX_VALUE, Long.MAX_VALUE));
+    seqFile3.getModFile().close();
+    seqResources.add(seqFile1);
+    seqResources.add(seqFile2);
+    seqResources.add(seqFile3);
+    InnerSpaceCompactionTask task =
+        new InnerSpaceCompactionTask(
+            0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0);
+    Assert.assertTrue(task.start());
+    Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+  }
+
+  @Test
+  public void testReadChunkPerformerWithEmptyTargetFile4()
+      throws IOException, IllegalPathException {
+    TsFileResource seqFile1 = createEmptyFileAndResource(true);
+    try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqFile1)) {
+      writer.startChunkGroup("d1");
+      writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+          "s1", new TimeRange[] {new TimeRange(1, 2)}, TSEncoding.PLAIN, CompressionType.LZ4);
+      writer.endChunkGroup();
+      writer.endFile();
+    }
+    seqFile1
+        .getModFile()
+        .write(new Deletion(new PartialPath("root.testsg.d1.**"), Long.MAX_VALUE, Long.MAX_VALUE));
+    seqFile1.getModFile().close();
+    TsFileResource seqFile2 = createEmptyFileAndResource(true);
+    try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqFile2)) {
+      writer.startChunkGroup("d1");
+      writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+          "s1", new TimeRange[] {new TimeRange(5, 6)}, TSEncoding.PLAIN, CompressionType.LZ4);
+      writer.endChunkGroup();
+      writer.endFile();
+    }
+    seqFile2
+        .getModFile()
+        .write(new Deletion(new PartialPath("root.testsg.d1.**"), Long.MAX_VALUE, Long.MAX_VALUE));
+    seqFile2.getModFile().close();
+    TsFileResource seqFile3 = createEmptyFileAndResource(true);
+    try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqFile3)) {
+      writer.startChunkGroup("d1");
+      writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+          "s1", new TimeRange[] {new TimeRange(10, 20)}, TSEncoding.PLAIN, CompressionType.LZ4);
+      writer.endChunkGroup();
+      writer.endFile();
+    }
+    seqFile3
+        .getModFile()
+        .write(new Deletion(new PartialPath("root.testsg.d1.**"), Long.MAX_VALUE, Long.MAX_VALUE));
+    seqFile3.getModFile().close();
+    seqResources.add(seqFile1);
+    seqResources.add(seqFile2);
+    seqResources.add(seqFile3);
+    InnerSpaceCompactionTask task =
+        new InnerSpaceCompactionTask(
+            0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0);
+    Assert.assertTrue(task.start());
+    Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+  }
+
   @Test
   public void testReadChunkPerformerWithNonEmptyTargetFile() throws IOException {
     TsFileResource seqFile1 = createEmptyFileAndResource(true);