You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/07/11 07:48:38 UTC

(iotdb) branch rc/1.3.2 updated (abccd8fe5df -> cb524977371)

This is an automated email from the ASF dual-hosted git repository.

haonan pushed a change to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git


    from abccd8fe5df update RELEASE_NOTES.md
     new 36a18f127a4 Remove usage of CompressionRatio in compaction (#12875)
     new 5422add3988 Fix negative compression ratios after restart (#12879)
     new 5f9291c9746 modify compaction schedule (#12826)
     new b6aa1a872ad Limit the row count of a single TVList to avg_series_point_number_threshold (#12898)
     new 5167cb95bd1 spotless
     new cb524977371 CI

The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .github/workflows/cluster-it-1c1d.yml              |  2 +-
 .github/workflows/cluster-it-1c3d.yml              |  2 +-
 .github/workflows/pipe-it-2cluster.yml             |  2 +-
 .github/workflows/unit-test.yml                    |  2 +-
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |  2 +-
 .../iotdb/db/storageengine/StorageEngine.java      |  6 +-
 .../db/storageengine/dataregion/DataRegion.java    |  8 +--
 .../compaction/schedule/CompactionScheduler.java   | 20 ++++--
 .../compaction/schedule/CompactionTaskManager.java |  8 +++
 .../estimator/AbstractCompactionEstimator.java     |  6 +-
 .../FastCompactionInnerCompactionEstimator.java    | 15 +++--
 .../FastCrossSpaceCompactionEstimator.java         | 15 +++--
 .../ReadChunkInnerCompactionEstimator.java         | 19 +++---
 .../RepairUnsortedFileCompactionEstimator.java     | 11 ++--
 .../dataregion/flush/CompressionRatio.java         |  7 +++
 .../dataregion/memtable/AbstractMemTable.java      | 16 +++--
 .../memtable/AlignedWritableMemChunk.java          | 25 ++++----
 .../dataregion/memtable/IMemTable.java             |  7 +--
 .../dataregion/memtable/IWritableMemChunk.java     | 20 +++---
 .../dataregion/memtable/TsFileProcessor.java       | 25 +++++---
 .../dataregion/memtable/WritableMemChunk.java      | 71 +++++++++++-----------
 .../file/UnsealedTsFileRecoverPerformer.java       | 19 ++++++
 .../db/utils/datastructure/AlignedTVList.java      |  4 +-
 .../iotdb/db/utils/datastructure/BinaryTVList.java |  4 +-
 .../iotdb/db/utils/datastructure/TVList.java       |  6 +-
 ...eCompactionWithFastPerformerValidationTest.java |  4 ++
 ...actionWithReadPointPerformerValidationTest.java |  4 ++
 27 files changed, 189 insertions(+), 141 deletions(-)


(iotdb) 04/06: Limit the row count of a single TVList to avg_series_point_number_threshold (#12898)

Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit b6aa1a872ad4ddad433a0ddfa1fc53ad6beb44b6
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Jul 11 10:23:25 2024 +0800

    Limit the row count of a single TVList to avg_series_point_number_threshold (#12898)
    
    * init
    
    * optimize mem control code and flush metric
---
 .../iotdb/db/storageengine/StorageEngine.java      |  6 +-
 .../db/storageengine/dataregion/DataRegion.java    |  8 +--
 .../dataregion/memtable/AbstractMemTable.java      | 16 +++--
 .../memtable/AlignedWritableMemChunk.java          | 25 ++++----
 .../dataregion/memtable/IMemTable.java             |  7 +--
 .../dataregion/memtable/IWritableMemChunk.java     | 20 +++---
 .../dataregion/memtable/TsFileProcessor.java       | 28 ++++++---
 .../dataregion/memtable/WritableMemChunk.java      | 71 +++++++++++-----------
 .../db/utils/datastructure/AlignedTVList.java      |  4 +-
 .../iotdb/db/utils/datastructure/BinaryTVList.java |  4 +-
 .../iotdb/db/utils/datastructure/TVList.java       |  6 +-
 11 files changed, 98 insertions(+), 97 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index f7737201f19..71145964ebb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -161,13 +161,9 @@ public class StorageEngine implements IService {
   }
 
   /** block insertion if the insertion is rejected by memory control */
-  public static void blockInsertionIfReject(TsFileProcessor tsFileProcessor)
-      throws WriteProcessRejectException {
+  public static void blockInsertionIfReject() throws WriteProcessRejectException {
     long startTime = System.currentTimeMillis();
     while (SystemInfo.getInstance().isRejected()) {
-      if (tsFileProcessor != null && tsFileProcessor.shouldFlush()) {
-        break;
-      }
       try {
         TimeUnit.MILLISECONDS.sleep(CONFIG.getCheckPeriodWhenInsertBlocked());
         if (System.currentTimeMillis() - startTime > CONFIG.getMaxWaitingTimeWhenInsertBlocked()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 55e94d07375..d3a4ea1fcfe 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -861,7 +861,7 @@ public class DataRegion implements IDataRegionForQuery {
       throw new OutOfTTLException(
           insertRowNode.getTime(), (CommonDateTimeUtils.currentTime() - dataTTL));
     }
-    StorageEngine.blockInsertionIfReject(null);
+    StorageEngine.blockInsertionIfReject();
     long startTime = System.nanoTime();
     writeLock("InsertRow");
     PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
@@ -910,7 +910,7 @@ public class DataRegion implements IDataRegionForQuery {
   @SuppressWarnings({"squid:S3776", "squid:S6541"}) // Suppress high Cognitive Complexity warning
   public void insertTablet(InsertTabletNode insertTabletNode)
       throws BatchProcessException, WriteProcessException {
-    StorageEngine.blockInsertionIfReject(null);
+    StorageEngine.blockInsertionIfReject();
     long startTime = System.nanoTime();
     writeLock("insertTablet");
     PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
@@ -3094,7 +3094,7 @@ public class DataRegion implements IDataRegionForQuery {
    */
   public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode)
       throws WriteProcessException, BatchProcessException {
-    StorageEngine.blockInsertionIfReject(null);
+    StorageEngine.blockInsertionIfReject();
     long startTime = System.nanoTime();
     writeLock("InsertRowsOfOneDevice");
     PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
@@ -3204,7 +3204,7 @@ public class DataRegion implements IDataRegionForQuery {
 
   public void insert(InsertRowsNode insertRowsNode)
       throws BatchProcessException, WriteProcessRejectException {
-    StorageEngine.blockInsertionIfReject(null);
+    StorageEngine.blockInsertionIfReject();
     long startTime = System.nanoTime();
     writeLock("InsertRows");
     PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index 07f65e229f8..e898736485e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -69,6 +69,7 @@ public abstract class AbstractMemTable implements IMemTable {
   private static final DeviceIDFactory deviceIDFactory = DeviceIDFactory.getInstance();
 
   private boolean shouldFlush = false;
+  private boolean reachChunkSizeOrPointNumThreshold = false;
   private volatile FlushStatus flushStatus = FlushStatus.WORKING;
   private final int avgSeriesPointNumThreshold =
       IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
@@ -318,7 +319,7 @@ public abstract class AbstractMemTable implements IMemTable {
     IWritableMemChunkGroup memChunkGroup =
         createMemChunkGroupIfNotExistAndGet(deviceId, schemaList);
     if (memChunkGroup.writeWithFlushCheck(insertTime, objectValue, schemaList)) {
-      shouldFlush = true;
+      reachChunkSizeOrPointNumThreshold = true;
     }
   }
 
@@ -331,7 +332,7 @@ public abstract class AbstractMemTable implements IMemTable {
     IWritableMemChunkGroup memChunkGroup =
         createAlignedMemChunkGroupIfNotExistAndGet(deviceId, schemaList);
     if (memChunkGroup.writeWithFlushCheck(insertTime, objectValue, schemaList)) {
-      shouldFlush = true;
+      reachChunkSizeOrPointNumThreshold = true;
     }
   }
 
@@ -353,7 +354,7 @@ public abstract class AbstractMemTable implements IMemTable {
         schemaList,
         start,
         end)) {
-      shouldFlush = true;
+      reachChunkSizeOrPointNumThreshold = true;
     }
   }
 
@@ -379,7 +380,7 @@ public abstract class AbstractMemTable implements IMemTable {
         schemaList,
         start,
         end)) {
-      shouldFlush = true;
+      reachChunkSizeOrPointNumThreshold = true;
     }
   }
 
@@ -426,11 +427,8 @@ public abstract class AbstractMemTable implements IMemTable {
   }
 
   @Override
-  public boolean reachTotalPointNumThreshold() {
-    if (totalPointsNum == 0) {
-      return false;
-    }
-    return totalPointsNum >= totalPointsNumThreshold;
+  public boolean reachChunkSizeOrPointNumThreshold() {
+    return reachChunkSizeOrPointNumThreshold;
   }
 
   @Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 489520b201c..934b616e393 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -87,22 +87,22 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
   }
 
   @Override
-  public void putLong(long t, long v) {
+  public boolean putLongWithFlushCheck(long t, long v) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
   @Override
-  public void putInt(long t, int v) {
+  public boolean putIntWithFlushCheck(long t, int v) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
   @Override
-  public void putFloat(long t, float v) {
+  public boolean putFloatWithFlushCheck(long t, float v) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
   @Override
-  public void putDouble(long t, double v) {
+  public boolean putDoubleWithFlushCheck(long t, double v) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
@@ -112,33 +112,33 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
   }
 
   @Override
-  public void putBoolean(long t, boolean v) {
+  public boolean putBooleanWithFlushCheck(long t, boolean v) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
   @Override
   public boolean putAlignedValueWithFlushCheck(long t, Object[] v) {
     list.putAlignedValue(t, v);
-    return list.reachMaxChunkSizeThreshold();
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end) {
+  public boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int start, int end) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
   @Override
-  public void putInts(long[] t, int[] v, BitMap bitMap, int start, int end) {
+  public boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int start, int end) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
   @Override
-  public void putFloats(long[] t, float[] v, BitMap bitMap, int start, int end) {
+  public boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap, int start, int end) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
   @Override
-  public void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int end) {
+  public boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap, int start, int end) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
@@ -149,7 +149,8 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
   }
 
   @Override
-  public void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end) {
+  public boolean putBooleansWithFlushCheck(
+      long[] t, boolean[] v, BitMap bitMap, int start, int end) {
     throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
   }
 
@@ -157,7 +158,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
   public boolean putAlignedValuesWithFlushCheck(
       long[] t, Object[] v, BitMap[] bitMaps, int start, int end) {
     list.putAlignedValues(t, v, bitMaps, start, end);
-    return list.reachMaxChunkSizeThreshold();
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
index b2288f174f8..222b4199105 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
@@ -75,12 +75,7 @@ public interface IMemTable extends WALEntryValue {
   /** only used when mem control enabled */
   long getTVListsRamCost();
 
-  /**
-   * only used when mem control enabled
-   *
-   * @return whether the average number of points in each WritableChunk reaches the threshold
-   */
-  boolean reachTotalPointNumThreshold();
+  boolean reachChunkSizeOrPointNumThreshold();
 
   int getSeriesNumber();
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
index 4c07c3ece33..fea1770fdee 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
@@ -30,31 +30,31 @@ import java.util.List;
 
 public interface IWritableMemChunk extends WALEntryValue {
 
-  void putLong(long t, long v);
+  boolean putLongWithFlushCheck(long t, long v);
 
-  void putInt(long t, int v);
+  boolean putIntWithFlushCheck(long t, int v);
 
-  void putFloat(long t, float v);
+  boolean putFloatWithFlushCheck(long t, float v);
 
-  void putDouble(long t, double v);
+  boolean putDoubleWithFlushCheck(long t, double v);
 
   boolean putBinaryWithFlushCheck(long t, Binary v);
 
-  void putBoolean(long t, boolean v);
+  boolean putBooleanWithFlushCheck(long t, boolean v);
 
   boolean putAlignedValueWithFlushCheck(long t, Object[] v);
 
-  void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end);
+  boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int start, int end);
 
-  void putInts(long[] t, int[] v, BitMap bitMap, int start, int end);
+  boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int start, int end);
 
-  void putFloats(long[] t, float[] v, BitMap bitMap, int start, int end);
+  boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap, int start, int end);
 
-  void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int end);
+  boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap, int start, int end);
 
   boolean putBinariesWithFlushCheck(long[] t, Binary[] v, BitMap bitMap, int start, int end);
 
-  void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end);
+  boolean putBooleansWithFlushCheck(long[] t, boolean[] v, BitMap bitMap, int start, int end);
 
   boolean putAlignedValuesWithFlushCheck(
       long[] t, Object[] v, BitMap[] bitMaps, int start, int end);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 680f30389b5..6233b426b3d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -43,7 +43,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNo
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
 import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils;
 import org.apache.iotdb.db.service.metrics.WritingMetrics;
-import org.apache.iotdb.db.storageengine.StorageEngine;
 import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
 import org.apache.iotdb.db.storageengine.dataregion.DataRegionInfo;
 import org.apache.iotdb.db.storageengine.dataregion.flush.CloseFileListener;
@@ -91,6 +90,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
@@ -909,7 +909,22 @@ public class TsFileProcessor {
     if (dataRegionInfo.needToReportToSystem()) {
       try {
         if (!SystemInfo.getInstance().reportStorageGroupStatus(dataRegionInfo, this)) {
-          StorageEngine.blockInsertionIfReject(this);
+          long startTime = System.currentTimeMillis();
+          while (SystemInfo.getInstance().isRejected()) {
+            if (workMemTable.shouldFlush()) {
+              break;
+            }
+            try {
+              TimeUnit.MILLISECONDS.sleep(config.getCheckPeriodWhenInsertBlocked());
+              if (System.currentTimeMillis() - startTime
+                  > config.getMaxWaitingTimeWhenInsertBlocked()) {
+                throw new WriteProcessRejectException(
+                    "System rejected over " + (System.currentTimeMillis() - startTime) + "ms");
+              }
+            } catch (InterruptedException e) {
+              Thread.currentThread().interrupt();
+            }
+          }
         }
       } catch (WriteProcessRejectException e) {
         dataRegionInfo.releaseStorageGroupMemCost(memTableIncrement);
@@ -987,13 +1002,8 @@ public class TsFileProcessor {
     if (workMemTable.shouldFlush()) {
       return true;
     }
-    if (workMemTable.reachTotalPointNumThreshold()) {
-      logger.info(
-          "The avg series points num {} of tsfile {} reaches the threshold",
-          workMemTable.getTotalPointsNum() / workMemTable.getSeriesNumber(),
-          tsFileResource.getTsFile().getAbsolutePath());
-      WritingMetrics.getInstance()
-          .recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
+    if (workMemTable.reachChunkSizeOrPointNumThreshold()) {
+      WritingMetrics.getInstance().recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
       return true;
     }
     return false;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index c1a71d32a53..134bf4310e9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -61,26 +61,20 @@ public class WritableMemChunk implements IWritableMemChunk {
   public boolean writeWithFlushCheck(long insertTime, Object objectValue) {
     switch (schema.getType()) {
       case BOOLEAN:
-        putBoolean(insertTime, (boolean) objectValue);
-        break;
+        return putBooleanWithFlushCheck(insertTime, (boolean) objectValue);
       case INT32:
-        putInt(insertTime, (int) objectValue);
-        break;
+        return putIntWithFlushCheck(insertTime, (int) objectValue);
       case INT64:
-        putLong(insertTime, (long) objectValue);
-        break;
+        return putLongWithFlushCheck(insertTime, (long) objectValue);
       case FLOAT:
-        putFloat(insertTime, (float) objectValue);
-        break;
+        return putFloatWithFlushCheck(insertTime, (float) objectValue);
       case DOUBLE:
-        putDouble(insertTime, (double) objectValue);
-        break;
+        return putDoubleWithFlushCheck(insertTime, (double) objectValue);
       case TEXT:
         return putBinaryWithFlushCheck(insertTime, (Binary) objectValue);
       default:
-        throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + schema.getType());
+        throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + schema.getType().name());
     }
-    return false;
   }
 
   @Override
@@ -95,31 +89,25 @@ public class WritableMemChunk implements IWritableMemChunk {
     switch (dataType) {
       case BOOLEAN:
         boolean[] boolValues = (boolean[]) valueList;
-        putBooleans(times, boolValues, bitMap, start, end);
-        break;
+        return putBooleansWithFlushCheck(times, boolValues, bitMap, start, end);
       case INT32:
         int[] intValues = (int[]) valueList;
-        putInts(times, intValues, bitMap, start, end);
-        break;
+        return putIntsWithFlushCheck(times, intValues, bitMap, start, end);
       case INT64:
         long[] longValues = (long[]) valueList;
-        putLongs(times, longValues, bitMap, start, end);
-        break;
+        return putLongsWithFlushCheck(times, longValues, bitMap, start, end);
       case FLOAT:
         float[] floatValues = (float[]) valueList;
-        putFloats(times, floatValues, bitMap, start, end);
-        break;
+        return putFloatsWithFlushCheck(times, floatValues, bitMap, start, end);
       case DOUBLE:
         double[] doubleValues = (double[]) valueList;
-        putDoubles(times, doubleValues, bitMap, start, end);
-        break;
+        return putDoublesWithFlushCheck(times, doubleValues, bitMap, start, end);
       case TEXT:
         Binary[] binaryValues = (Binary[]) valueList;
         return putBinariesWithFlushCheck(times, binaryValues, bitMap, start, end);
       default:
-        throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType);
+        throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType.name());
     }
-    return false;
   }
 
   @Override
@@ -134,34 +122,39 @@ public class WritableMemChunk implements IWritableMemChunk {
   }
 
   @Override
-  public void putLong(long t, long v) {
+  public boolean putLongWithFlushCheck(long t, long v) {
     list.putLong(t, v);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putInt(long t, int v) {
+  public boolean putIntWithFlushCheck(long t, int v) {
     list.putInt(t, v);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putFloat(long t, float v) {
+  public boolean putFloatWithFlushCheck(long t, float v) {
     list.putFloat(t, v);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putDouble(long t, double v) {
+  public boolean putDoubleWithFlushCheck(long t, double v) {
     list.putDouble(t, v);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
   public boolean putBinaryWithFlushCheck(long t, Binary v) {
     list.putBinary(t, v);
-    return list.reachMaxChunkSizeThreshold();
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putBoolean(long t, boolean v) {
+  public boolean putBooleanWithFlushCheck(long t, boolean v) {
     list.putBoolean(t, v);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
@@ -170,35 +163,41 @@ public class WritableMemChunk implements IWritableMemChunk {
   }
 
   @Override
-  public void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end) {
+  public boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int start, int end) {
     list.putLongs(t, v, bitMap, start, end);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putInts(long[] t, int[] v, BitMap bitMap, int start, int end) {
+  public boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int start, int end) {
     list.putInts(t, v, bitMap, start, end);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putFloats(long[] t, float[] v, BitMap bitMap, int start, int end) {
+  public boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap, int start, int end) {
     list.putFloats(t, v, bitMap, start, end);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int end) {
+  public boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap, int start, int end) {
     list.putDoubles(t, v, bitMap, start, end);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
   public boolean putBinariesWithFlushCheck(
       long[] t, Binary[] v, BitMap bitMap, int start, int end) {
     list.putBinaries(t, v, bitMap, start, end);
-    return list.reachMaxChunkSizeThreshold();
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
-  public void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end) {
+  public boolean putBooleansWithFlushCheck(
+      long[] t, boolean[] v, BitMap bitMap, int start, int end) {
     list.putBooleans(t, v, bitMap, start, end);
+    return list.reachChunkSizeOrPointNumThreshold();
   }
 
   @Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 4092db7d442..4949044d590 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -698,8 +698,8 @@ public abstract class AlignedTVList extends TVList {
   }
 
   @Override
-  public boolean reachMaxChunkSizeThreshold() {
-    return reachMaxChunkSizeFlag;
+  public boolean reachChunkSizeOrPointNumThreshold() {
+    return reachMaxChunkSizeFlag || rowCount >= MAX_SERIES_POINT_NUMBER;
   }
 
   @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
index 4c5d4061530..bce5dc083c3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
@@ -98,8 +98,8 @@ public abstract class BinaryTVList extends TVList {
   }
 
   @Override
-  public boolean reachMaxChunkSizeThreshold() {
-    return memoryBinaryChunkSize >= TARGET_CHUNK_SIZE;
+  public boolean reachChunkSizeOrPointNumThreshold() {
+    return memoryBinaryChunkSize >= TARGET_CHUNK_SIZE || rowCount >= MAX_SERIES_POINT_NUMBER;
   }
 
   @Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index f2b26158a5f..f6c34ba0d62 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -49,6 +49,8 @@ public abstract class TVList implements WALEntryValue {
   protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not consistent";
   protected static final long TARGET_CHUNK_SIZE =
       IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+  protected static final long MAX_SERIES_POINT_NUMBER =
+      IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
   // list of timestamp array, add 1 when expanded -> data point timestamp array
   // index relation: arrayIndex -> elementIndex
   protected List<long[]> timestamps;
@@ -148,8 +150,8 @@ public abstract class TVList implements WALEntryValue {
     throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
   }
 
-  public boolean reachMaxChunkSizeThreshold() {
-    return false;
+  public boolean reachChunkSizeOrPointNumThreshold() {
+    return rowCount >= MAX_SERIES_POINT_NUMBER;
   }
 
   public void putBoolean(long time, boolean value) {


(iotdb) 01/06: Remove usage of CompressionRatio in compaction (#12875)

Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 36a18f127a4caf2260d0143ddcf756573b49b775
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Mon Jul 8 22:12:18 2024 +0800

    Remove usage of CompressionRatio in compaction (#12875)
    
    * add log
    
    * add default value
    
    * remove empty line
    
    * set old file name
    
    * remove usage of CompressionRatio in compaction
    
    * modify calculatingDataMemoryCost
    
    * fix ut
---
 .../estimator/AbstractCompactionEstimator.java        |  6 +++---
 .../FastCompactionInnerCompactionEstimator.java       | 15 +++++++--------
 .../estimator/FastCrossSpaceCompactionEstimator.java  | 15 +++++++--------
 .../estimator/ReadChunkInnerCompactionEstimator.java  | 19 +++++++++----------
 .../RepairUnsortedFileCompactionEstimator.java        | 11 ++++-------
 .../dataregion/flush/CompressionRatio.java            |  7 +++++++
 ...paceCompactionWithFastPerformerValidationTest.java |  4 ++++
 ...ompactionWithReadPointPerformerValidationTest.java |  4 ++++
 8 files changed, 45 insertions(+), 36 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
index e72f163a21c..6c28e56a5c1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
@@ -21,12 +21,13 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimat
 
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
 import org.apache.iotdb.tsfile.file.metadata.IDeviceID;
 import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
 
@@ -58,8 +59,7 @@ public abstract class AbstractCompactionEstimator {
   protected Map<TsFileResource, DeviceTimeIndex> deviceTimeIndexCache = new HashMap<>();
 
   protected IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
-
-  protected long compressionRatio = (long) CompressionRatio.getInstance().getRatio() + 1;
+  protected TSFileConfig tsFileConfig = TSFileDescriptor.getInstance().getConfig();
 
   protected abstract long calculatingMetadataMemoryCost(CompactionTaskInfo taskInfo);
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
index 8be9b210a52..8530219671e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
@@ -55,21 +55,20 @@ public class FastCompactionInnerCompactionEstimator extends AbstractInnerSpaceEs
     }
     long maxConcurrentSeriesNum =
         Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum());
-    long averageUncompressedChunkSize =
-        taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+    long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
 
     long maxConcurrentSeriesSizeOfTotalFiles =
-        averageUncompressedChunkSize
-            * taskInfo.getFileInfoList().size()
-            * maxConcurrentSeriesNum
-            * taskInfo.getMaxChunkMetadataNumInSeries()
-            / compressionRatio;
+        averageChunkSize
+                * taskInfo.getFileInfoList().size()
+                * maxConcurrentSeriesNum
+                * taskInfo.getMaxChunkMetadataNumInSeries()
+            + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
     long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
     long targetChunkWriterSize =
         Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize);
 
     long maxConcurrentChunkSizeFromSourceFile =
-        averageUncompressedChunkSize
+        (averageChunkSize + tsFileConfig.getPageSizeInByte())
             * maxConcurrentSeriesNum
             * calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources());
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
index 1ac3b5d5be1..4d9b89fda52 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
@@ -56,21 +56,20 @@ public class FastCrossSpaceCompactionEstimator extends AbstractCrossSpaceEstimat
 
     long maxConcurrentSeriesNum =
         Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum());
-    long averageUncompressedChunkSize =
-        taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+    long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
 
     long maxConcurrentSeriesSizeOfTotalFiles =
-        averageUncompressedChunkSize
-            * taskInfo.getFileInfoList().size()
-            * maxConcurrentSeriesNum
-            * taskInfo.getMaxChunkMetadataNumInSeries()
-            / compressionRatio;
+        averageChunkSize
+                * taskInfo.getFileInfoList().size()
+                * maxConcurrentSeriesNum
+                * taskInfo.getMaxChunkMetadataNumInSeries()
+            + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
     long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
     long targetChunkWriterSize =
         Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize);
 
     long maxConcurrentChunkSizeFromSourceFile =
-        averageUncompressedChunkSize
+        (averageChunkSize + tsFileConfig.getPageSizeInByte())
             * maxConcurrentSeriesNum
             * calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources());
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
index e432b1eac5d..e77a87f4c58 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
@@ -51,22 +51,21 @@ public class ReadChunkInnerCompactionEstimator extends AbstractInnerSpaceEstimat
     if (taskInfo.getTotalChunkNum() == 0) {
       return taskInfo.getModificationFileSize();
     }
-    long averageUncompressedChunkSize =
-        taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+    long maxConcurrentSeriesNum = taskInfo.getMaxConcurrentSeriesNum();
+    long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
 
     long maxConcurrentSeriesSizeOfTotalFiles =
-        averageUncompressedChunkSize
-            * taskInfo.getFileInfoList().size()
-            * taskInfo.getMaxConcurrentSeriesNum()
-            * taskInfo.getMaxChunkMetadataNumInSeries()
-            / compressionRatio;
-    long maxTargetChunkWriterSize =
-        config.getTargetChunkSize() * taskInfo.getMaxConcurrentSeriesNum();
+        averageChunkSize
+                * taskInfo.getFileInfoList().size()
+                * maxConcurrentSeriesNum
+                * taskInfo.getMaxChunkMetadataNumInSeries()
+            + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
+    long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
     long targetChunkWriterSize =
         Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize);
 
     long chunkSizeFromSourceFile =
-        averageUncompressedChunkSize * taskInfo.getMaxConcurrentSeriesNum();
+        (averageChunkSize + tsFileConfig.getPageSizeInByte()) * maxConcurrentSeriesNum;
 
     return targetChunkWriterSize + chunkSizeFromSourceFile + taskInfo.getModificationFileSize();
   }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
index 3eadd41ea28..c7852753822 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
@@ -50,19 +50,16 @@ public class RepairUnsortedFileCompactionEstimator extends AbstractInnerSpaceEst
     }
     long maxConcurrentSeriesNum =
         Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum());
-    long averageUncompressedChunkSize =
-        taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+    long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
 
     long maxConcurrentSeriesSize =
-        averageUncompressedChunkSize
-            * maxConcurrentSeriesNum
-            * taskInfo.getMaxChunkMetadataNumInSeries()
-            / compressionRatio;
+        averageChunkSize * maxConcurrentSeriesNum * taskInfo.getMaxChunkMetadataNumInSeries()
+            + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
     long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
     long targetChunkWriterSize = Math.min(maxConcurrentSeriesSize, maxTargetChunkWriterSize);
 
     long inMemorySortedDataSize =
-        averageUncompressedChunkSize
+        (averageChunkSize + tsFileConfig.getPageSizeInByte())
             * Math.min(
                 taskInfo.getMaxChunkMetadataNumInDevice(),
                 taskInfo.getMaxChunkMetadataNumInSeries() * maxConcurrentSeriesNum);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
index 5d6c40e1487..2f1e173f249 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
@@ -85,6 +85,12 @@ public class CompressionRatio {
    * call this method.
    */
   public synchronized void updateRatio(long memorySize, long diskSize) throws IOException {
+    if (memorySize < 0 || totalMemorySize.get() < 0) {
+      LOGGER.warn(
+          "The compression ratio is negative, current memTableSize: {}, totalMemTableSize: {}",
+          memorySize,
+          totalMemorySize);
+    }
     File oldFile = SystemFileFactory.INSTANCE.getFile(directory, oldFileName);
 
     totalMemorySize.addAndGet(memorySize);
@@ -149,6 +155,7 @@ public class CompressionRatio {
           "After restoring from compression ratio file, total memory size = {}, total disk size = {}",
           totalMemorySize,
           totalDiskSize);
+      oldFileName = ratioFiles[maxRatioIndex].getName();
       deleteRedundantFilesByIndex(ratioFiles, maxRatioIndex);
     } else { // If there is no new file, try to restore from the old version file
       File[] ratioFilesBeforeV121 =
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index d1bf1564d2a..cf9358340f6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
 import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
 import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
 import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -79,6 +80,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
   private final String oldThreadName = Thread.currentThread().getName();
 
   private ICrossCompactionPerformer performer = new FastCompactionPerformer(true);
+  private long compactionMemory = SystemInfo.getInstance().getMemorySizeForCompaction();
 
   @Before
   public void setUp()
@@ -87,6 +89,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
     IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
     IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
     TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+    SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024);
     Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
   }
 
@@ -95,6 +98,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
     super.tearDown();
     Thread.currentThread().setName(oldThreadName);
     FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+    SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory);
     TsFileValidationTool.badFileNum = 0;
   }
 
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
index c1090e39450..5c0edd7bc24 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
@@ -44,6 +44,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionF
 import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
 import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
 import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
 import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -77,6 +78,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
   private final String oldThreadName = Thread.currentThread().getName();
 
   private ICrossCompactionPerformer performer = new ReadPointCompactionPerformer();
+  private long compactionMemory = SystemInfo.getInstance().getMemorySizeForCompaction();
 
   @Before
   public void setUp()
@@ -85,6 +87,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
     IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
     IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
     TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+    SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024);
     Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
   }
 
@@ -93,6 +96,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
     super.tearDown();
     Thread.currentThread().setName(oldThreadName);
     FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+    SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory);
     TsFileValidationTool.badFileNum = 0;
   }
 


(iotdb) 02/06: Fix negative compression ratios after restart (#12879)

Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 5422add39882b1741b11b287c5404569e4a98a8c
Author: Haonan <hh...@outlook.com>
AuthorDate: Mon Jul 8 22:40:10 2024 +0800

    Fix negative compression ratios after restart (#12879)
---
 .../recover/file/UnsealedTsFileRecoverPerformer.java  | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
index 080867b8800..705027ab7d5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.pipe.agent.PipeAgent;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
+import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
 import org.apache.iotdb.db.storageengine.dataregion.flush.MemTableFlushTask;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunk;
@@ -259,6 +260,24 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform
         // set recover progress index for pipe
         PipeAgent.runtime().assignProgressIndexForTsFileRecovery(tsFileResource);
 
+        try {
+          long memTableSize = recoveryMemTable.memSize();
+          double compressionRatio = ((double) memTableSize) / writer.getPos();
+          logger.info(
+              "The compression ratio of tsfile {} is {}, totalMemTableSize: {}, the file size: {}",
+              writer.getFile().getAbsolutePath(),
+              String.format("%.2f", compressionRatio),
+              memTableSize,
+              writer.getPos());
+          CompressionRatio.getInstance().updateRatio(memTableSize, writer.getPos());
+        } catch (IOException e) {
+          logger.error(
+              "{}: {} update compression ratio failed",
+              databaseName,
+              tsFileResource.getTsFile().getName(),
+              e);
+        }
+
         // if we put following codes in the 'if' clause above, this file can be continued writing
         // into it
         // currently, we close this file anyway


(iotdb) 03/06: modify compaction schedule (#12826)

Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 5f9291c9746dfaaf207cd9b09b1b7a2bd417b1dc
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Thu Jul 4 13:10:34 2024 +0800

    modify compaction schedule (#12826)
---
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java   |  2 +-
 .../compaction/schedule/CompactionScheduler.java     | 20 +++++++++++++++-----
 .../compaction/schedule/CompactionTaskManager.java   |  8 ++++++++
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 1dd9c0c3942..6c870287b56 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -470,7 +470,7 @@ public class IoTDBConfig {
    * cross space compaction, eliminate the unsequence files first BALANCE: alternate two compaction
    * types
    */
-  private CompactionPriority compactionPriority = CompactionPriority.BALANCE;
+  private CompactionPriority compactionPriority = CompactionPriority.INNER_CROSS;
 
   private double chunkMetadataSizeProportion = 0.1;
 
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
index 8d57caee8fd..097caad018a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
@@ -100,11 +100,18 @@ public class CompactionScheduler {
     // the name of this variable is trySubmitCount, because the task submitted to the queue could be
     // evicted due to the low priority of the task
     int trySubmitCount = 0;
-    trySubmitCount += tryToSubmitCrossSpaceCompactionTask(tsFileManager, timePartition, summary);
-    trySubmitCount +=
-        tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, true, summary);
-    trySubmitCount +=
-        tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, false, summary);
+    try {
+      trySubmitCount +=
+          tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, true, summary);
+      trySubmitCount +=
+          tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, false, summary);
+      trySubmitCount += tryToSubmitCrossSpaceCompactionTask(tsFileManager, timePartition, summary);
+    } catch (InterruptedException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOGGER.error("Meet error in compaction schedule.", e);
+    }
+
     return trySubmitCount;
   }
 
@@ -248,6 +255,9 @@ public class CompactionScheduler {
     if (!config.isEnableCrossSpaceCompaction()) {
       return 0;
     }
+    if (!CompactionTaskManager.getInstance().shouldSelectCrossSpaceCompactionTask()) {
+      return 0;
+    }
     String logicalStorageGroupName = tsFileManager.getStorageGroupName();
     String dataRegionId = tsFileManager.getDataRegionId();
     long compactionConfigVersionWhenSelectTask =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
index 78117ae9d41..a1d98016fd2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
@@ -243,6 +243,14 @@ public class CompactionTaskManager implements IService {
     return ServiceType.COMPACTION_SERVICE;
   }
 
+  public boolean shouldSelectCrossSpaceCompactionTask() {
+    // If the queue size accounts for less than 0.8 of the total capacity, select cross space
+    // compaction task
+    int waitingQueueRestSize =
+        candidateCompactionTaskQueue.getMaxSize() - candidateCompactionTaskQueue.size();
+    return 5 * waitingQueueRestSize >= candidateCompactionTaskQueue.size();
+  }
+
   public boolean isWaitingQueueFull() {
     return candidateCompactionTaskQueue.size() == candidateCompactionTaskQueue.getMaxSize();
   }


(iotdb) 06/06: CI

Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit cb524977371d17e78980be960f68a5ebdc86dbb8
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Jul 11 15:48:21 2024 +0800

    CI
---
 .github/workflows/cluster-it-1c1d.yml  | 2 +-
 .github/workflows/cluster-it-1c3d.yml  | 2 +-
 .github/workflows/pipe-it-2cluster.yml | 2 +-
 .github/workflows/unit-test.yml        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml
index e897770dabf..edf54ebcba3 100644
--- a/.github/workflows/cluster-it-1c1d.yml
+++ b/.github/workflows/cluster-it-1c1d.yml
@@ -12,7 +12,7 @@ on:
   pull_request:
     branches:
       - master
-      - 'rel/1.*'
+      - 'rc/*'
       - pipe-meta-sync
     paths-ignore:
       - 'docs/**'
diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml
index c2edca6b9d2..2091aaeb5c3 100644
--- a/.github/workflows/cluster-it-1c3d.yml
+++ b/.github/workflows/cluster-it-1c3d.yml
@@ -12,7 +12,7 @@ on:
   pull_request:
     branches:
       - master
-      - 'rel/1.*'
+      - 'rc/*'
       - pipe-meta-sync
     paths-ignore:
       - 'docs/**'
diff --git a/.github/workflows/pipe-it-2cluster.yml b/.github/workflows/pipe-it-2cluster.yml
index 3e2313a490d..619993accb3 100644
--- a/.github/workflows/pipe-it-2cluster.yml
+++ b/.github/workflows/pipe-it-2cluster.yml
@@ -11,7 +11,7 @@ on:
   pull_request:
     branches:
       - master
-      - 'rel/1.*'
+      - 'rc/*'
     paths-ignore:
       - 'docs/**'
       - 'site/**'
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 72594a79aea..c780cea8a15 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -15,7 +15,7 @@ on:
   pull_request:
     branches:
       - master
-      - 'rel/*'
+      - 'rc/*'
       - pipe-meta-sync
     paths-ignore:
       - 'docs/**'


(iotdb) 05/06: spotless

Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 5167cb95bd156866bd7b7e6f8d2713ccdcaee5ed
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Jul 11 15:47:32 2024 +0800

    spotless
---
 .../iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 6233b426b3d..b140b840cdf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -1003,7 +1003,8 @@ public class TsFileProcessor {
       return true;
     }
     if (workMemTable.reachChunkSizeOrPointNumThreshold()) {
-      WritingMetrics.getInstance().recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
+      WritingMetrics.getInstance()
+          .recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
       return true;
     }
     return false;