[iotdb] branch CompactionVersion created (now 53f4654)

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a change to branch CompactionVersion
in repository https://gitbox.apache.org/repos/asf/iotdb.git.


      at 53f4654  add compaction version in cache key

This branch includes the following new commits:

     new 53f4654  add compaction version in cache key

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[iotdb] 01/01: add compaction version in cache key

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch CompactionVersion
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 53f465474dc06bac55834dea4cf0415dfa3846bf
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Tue Nov 2 21:17:18 2021 +0800

    add compaction version in cache key
---
 .../db/engine/cache/TimeSeriesMetadataCache.java   | 10 ++++++---
 .../iotdb/db/engine/cache/ChunkCacheTest.java      |  6 ++++--
 .../compaction/inner/InnerCompactionCacheTest.java |  2 +-
 .../iotdb/tsfile/file/metadata/ChunkMetadata.java  | 12 +++++++++--
 .../apache/iotdb/tsfile/utils/FilePathUtils.java   | 24 ++++++++++++++++++++--
 5 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index cd5497e..247474b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -402,15 +402,18 @@ public class TimeSeriesMetadataCache {
     private final String filePath;
     private final String tsFilePrefixPath;
     private final long tsFileVersion;
+    // high 32 bit is compaction level, low 32 bit is merge count
+    private final long compactionVersion;
     private final String device;
     private final String measurement;
 
     public TimeSeriesMetadataCacheKey(String filePath, String device, String measurement) {
       this.filePath = filePath;
-      Pair<String, Long> tsFilePrefixPathAndTsFileVersionPair =
+      Pair<String, long[]> tsFilePrefixPathAndTsFileVersionPair =
           FilePathUtils.getTsFilePrefixPathAndTsFileVersionPair(filePath);
       this.tsFilePrefixPath = tsFilePrefixPathAndTsFileVersionPair.left;
-      this.tsFileVersion = tsFilePrefixPathAndTsFileVersionPair.right;
+      this.tsFileVersion = tsFilePrefixPathAndTsFileVersionPair.right[0];
+      this.compactionVersion = tsFilePrefixPathAndTsFileVersionPair.right[1];
       this.device = device;
       this.measurement = measurement;
     }
@@ -427,12 +430,13 @@ public class TimeSeriesMetadataCache {
       return Objects.equals(measurement, that.measurement)
           && Objects.equals(device, that.device)
           && tsFileVersion == that.tsFileVersion
+          && compactionVersion == that.compactionVersion
           && tsFilePrefixPath.equals(that.tsFilePrefixPath);
     }
 
     @Override
     public int hashCode() {
-      return Objects.hash(tsFilePrefixPath, tsFileVersion, device, measurement);
+      return Objects.hash(tsFilePrefixPath, tsFileVersion, compactionVersion, device, measurement);
     }
   }
 
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java
index 2b13dd7..3db886c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkCacheTest.java
@@ -49,7 +49,9 @@ import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR;
 import static org.junit.Assert.assertTrue;
@@ -122,8 +124,8 @@ public class ChunkCacheTest {
     Assert.assertEquals(chunk1.getHeader(), chunk2.getHeader());
     Assert.assertEquals(chunk1.getData(), chunk2.getData());
 
-    chunkMetadataKey.setFilePath(null);
     try {
+      chunkMetadataKey.setFilePath(null);
       chunkCache.get(chunkMetadataKey);
       fail();
     } catch (NullPointerException e) {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionCacheTest.java
index a1c1e39..3e051cb 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionCacheTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionCacheTest.java
@@ -128,8 +128,8 @@ public class InnerCompactionCacheTest extends InnerCompactionTest {
             CompactionTaskManager.currentTaskNum);
     sizeTieredCompactionTask.call();
 
-    firstChunkMetadata.setFilePath(null);
     try {
+      firstChunkMetadata.setFilePath(null);
       ChunkCache.getInstance().get(firstChunkMetadata);
       fail();
     } catch (NullPointerException e) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 33908b6..1683c8d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
 import org.apache.iotdb.tsfile.read.common.TimeRange;
 import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
 import org.apache.iotdb.tsfile.utils.FilePathUtils;
+import org.apache.iotdb.tsfile.utils.Pair;
 import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
 import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
 
@@ -78,6 +79,8 @@ public class ChunkMetadata implements Accountable, IChunkMetadata {
 
   // used for ChunkCache, Eg:"root.sg1/0/0"
   private String tsFilePrefixPath;
+  // high 32 bit is compaction level, low 32 bit is merge count
+  private long compactionVersion;
 
   public ChunkMetadata() {}
 
@@ -246,12 +249,13 @@ public class ChunkMetadata implements Accountable, IChunkMetadata {
     ChunkMetadata that = (ChunkMetadata) o;
     return offsetOfChunkHeader == that.offsetOfChunkHeader
         && version == that.version
+        && compactionVersion == that.compactionVersion
         && tsFilePrefixPath.equals(that.tsFilePrefixPath);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(tsFilePrefixPath, version, offsetOfChunkHeader);
+    return Objects.hash(tsFilePrefixPath, version, compactionVersion, offsetOfChunkHeader);
   }
 
   @Override
@@ -326,8 +330,12 @@ public class ChunkMetadata implements Accountable, IChunkMetadata {
   public void setFilePath(String filePath) {
     this.filePath = filePath;
 
+    Pair<String, long[]> tsFilePrefixPathAndTsFileVersionPair =
+        FilePathUtils.getTsFilePrefixPathAndTsFileVersionPair(filePath);
     // set tsFilePrefixPath
-    tsFilePrefixPath = FilePathUtils.getTsFilePrefixPath(filePath);
+    tsFilePrefixPath = tsFilePrefixPathAndTsFileVersionPair.left;
+    this.version = tsFilePrefixPathAndTsFileVersionPair.right[0];
+    this.compactionVersion = tsFilePrefixPathAndTsFileVersionPair.right[1];
   }
 
   @Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
index 87e8e03..ccf9399 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
 
 import java.io.File;
 
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
 public class FilePathUtils {
 
   private static final String PATH_SPLIT_STRING = File.separator.equals("\\") ? "\\\\" : "/";
@@ -103,6 +105,19 @@ public class FilePathUtils {
     return Long.parseLong(names[1]);
   }
 
+  public static long[] splitAndGetVersionArray(String tsFileName) {
+    String[] names = tsFileName.split(FILE_NAME_SEPARATOR);
+    long[] versionArray = new long[2];
+    if (names.length != 4) {
+      return versionArray;
+    }
+    versionArray[0] = Long.parseLong(names[1]);
+    versionArray[1] =
+        (Long.parseLong(names[2]) << 32)
+            | Long.parseLong(names[3].substring(0, names[3].length() - TSFILE_SUFFIX.length()));
+    return versionArray;
+  }
+
   public static Pair<String, Long> getLogicalSgNameAndTimePartitionIdPair(
       String tsFileAbsolutePath) {
     String[] pathSegments = splitTsFilePath(tsFileAbsolutePath);
@@ -111,7 +126,12 @@ public class FilePathUtils {
         Long.parseLong(pathSegments[pathSegments.length - 2]));
   }
 
-  public static Pair<String, Long> getTsFilePrefixPathAndTsFileVersionPair(
+  /**
+   * pair.left tsFilePrefixPath, like data/data/sequence/root.sg1/0/0 pair.right is a long array
+   * whose length is 2 pair.right[0] is tsfile version pair.right[1] is compaction version, high 32
+   * bit is compaction level, low 32 bit is merge count
+   */
+  public static Pair<String, long[]> getTsFilePrefixPathAndTsFileVersionPair(
       String tsFileAbsolutePath) {
     String[] pathSegments = splitTsFilePath(tsFileAbsolutePath);
     int pathLength = pathSegments.length;
@@ -121,6 +141,6 @@ public class FilePathUtils {
             + pathSegments[pathLength - 3]
             + File.separator
             + pathSegments[pathLength - 2],
-        splitAndGetTsFileVersion(pathSegments[pathLength - 1]));
+        splitAndGetVersionArray(pathSegments[pathLength - 1]));
   }
 }