You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/07/11 07:48:38 UTC
(iotdb) branch rc/1.3.2 updated (abccd8fe5df -> cb524977371)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from abccd8fe5df update RELEASE_NOTES.md
new 36a18f127a4 Remove usage of CompressionRatio in compaction (#12875)
new 5422add3988 Fix negative compression ratios after restart (#12879)
new 5f9291c9746 modify compaction schedule (#12826)
new b6aa1a872ad Limit the row count of a single TVList to avg_series_point_number_threshold (#12898)
new 5167cb95bd1 spotless
new cb524977371 CI
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/cluster-it-1c1d.yml | 2 +-
.github/workflows/cluster-it-1c3d.yml | 2 +-
.github/workflows/pipe-it-2cluster.yml | 2 +-
.github/workflows/unit-test.yml | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 6 +-
.../db/storageengine/dataregion/DataRegion.java | 8 +--
.../compaction/schedule/CompactionScheduler.java | 20 ++++--
.../compaction/schedule/CompactionTaskManager.java | 8 +++
.../estimator/AbstractCompactionEstimator.java | 6 +-
.../FastCompactionInnerCompactionEstimator.java | 15 +++--
.../FastCrossSpaceCompactionEstimator.java | 15 +++--
.../ReadChunkInnerCompactionEstimator.java | 19 +++---
.../RepairUnsortedFileCompactionEstimator.java | 11 ++--
.../dataregion/flush/CompressionRatio.java | 7 +++
.../dataregion/memtable/AbstractMemTable.java | 16 +++--
.../memtable/AlignedWritableMemChunk.java | 25 ++++----
.../dataregion/memtable/IMemTable.java | 7 +--
.../dataregion/memtable/IWritableMemChunk.java | 20 +++---
.../dataregion/memtable/TsFileProcessor.java | 25 +++++---
.../dataregion/memtable/WritableMemChunk.java | 71 +++++++++++-----------
.../file/UnsealedTsFileRecoverPerformer.java | 19 ++++++
.../db/utils/datastructure/AlignedTVList.java | 4 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 4 +-
.../iotdb/db/utils/datastructure/TVList.java | 6 +-
...eCompactionWithFastPerformerValidationTest.java | 4 ++
...actionWithReadPointPerformerValidationTest.java | 4 ++
27 files changed, 189 insertions(+), 141 deletions(-)
(iotdb) 04/06: Limit the row count of a single TVList to avg_series_point_number_threshold (#12898)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b6aa1a872ad4ddad433a0ddfa1fc53ad6beb44b6
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Jul 11 10:23:25 2024 +0800
Limit the row count of a single TVList to avg_series_point_number_threshold (#12898)
* init
* optimize mem control code and flush metric
---
.../iotdb/db/storageengine/StorageEngine.java | 6 +-
.../db/storageengine/dataregion/DataRegion.java | 8 +--
.../dataregion/memtable/AbstractMemTable.java | 16 +++--
.../memtable/AlignedWritableMemChunk.java | 25 ++++----
.../dataregion/memtable/IMemTable.java | 7 +--
.../dataregion/memtable/IWritableMemChunk.java | 20 +++---
.../dataregion/memtable/TsFileProcessor.java | 28 ++++++---
.../dataregion/memtable/WritableMemChunk.java | 71 +++++++++++-----------
.../db/utils/datastructure/AlignedTVList.java | 4 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 4 +-
.../iotdb/db/utils/datastructure/TVList.java | 6 +-
11 files changed, 98 insertions(+), 97 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index f7737201f19..71145964ebb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -161,13 +161,9 @@ public class StorageEngine implements IService {
}
/** block insertion if the insertion is rejected by memory control */
- public static void blockInsertionIfReject(TsFileProcessor tsFileProcessor)
- throws WriteProcessRejectException {
+ public static void blockInsertionIfReject() throws WriteProcessRejectException {
long startTime = System.currentTimeMillis();
while (SystemInfo.getInstance().isRejected()) {
- if (tsFileProcessor != null && tsFileProcessor.shouldFlush()) {
- break;
- }
try {
TimeUnit.MILLISECONDS.sleep(CONFIG.getCheckPeriodWhenInsertBlocked());
if (System.currentTimeMillis() - startTime > CONFIG.getMaxWaitingTimeWhenInsertBlocked()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 55e94d07375..d3a4ea1fcfe 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -861,7 +861,7 @@ public class DataRegion implements IDataRegionForQuery {
throw new OutOfTTLException(
insertRowNode.getTime(), (CommonDateTimeUtils.currentTime() - dataTTL));
}
- StorageEngine.blockInsertionIfReject(null);
+ StorageEngine.blockInsertionIfReject();
long startTime = System.nanoTime();
writeLock("InsertRow");
PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
@@ -910,7 +910,7 @@ public class DataRegion implements IDataRegionForQuery {
@SuppressWarnings({"squid:S3776", "squid:S6541"}) // Suppress high Cognitive Complexity warning
public void insertTablet(InsertTabletNode insertTabletNode)
throws BatchProcessException, WriteProcessException {
- StorageEngine.blockInsertionIfReject(null);
+ StorageEngine.blockInsertionIfReject();
long startTime = System.nanoTime();
writeLock("insertTablet");
PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
@@ -3094,7 +3094,7 @@ public class DataRegion implements IDataRegionForQuery {
*/
public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode)
throws WriteProcessException, BatchProcessException {
- StorageEngine.blockInsertionIfReject(null);
+ StorageEngine.blockInsertionIfReject();
long startTime = System.nanoTime();
writeLock("InsertRowsOfOneDevice");
PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
@@ -3204,7 +3204,7 @@ public class DataRegion implements IDataRegionForQuery {
public void insert(InsertRowsNode insertRowsNode)
throws BatchProcessException, WriteProcessRejectException {
- StorageEngine.blockInsertionIfReject(null);
+ StorageEngine.blockInsertionIfReject();
long startTime = System.nanoTime();
writeLock("InsertRows");
PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index 07f65e229f8..e898736485e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -69,6 +69,7 @@ public abstract class AbstractMemTable implements IMemTable {
private static final DeviceIDFactory deviceIDFactory = DeviceIDFactory.getInstance();
private boolean shouldFlush = false;
+ private boolean reachChunkSizeOrPointNumThreshold = false;
private volatile FlushStatus flushStatus = FlushStatus.WORKING;
private final int avgSeriesPointNumThreshold =
IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
@@ -318,7 +319,7 @@ public abstract class AbstractMemTable implements IMemTable {
IWritableMemChunkGroup memChunkGroup =
createMemChunkGroupIfNotExistAndGet(deviceId, schemaList);
if (memChunkGroup.writeWithFlushCheck(insertTime, objectValue, schemaList)) {
- shouldFlush = true;
+ reachChunkSizeOrPointNumThreshold = true;
}
}
@@ -331,7 +332,7 @@ public abstract class AbstractMemTable implements IMemTable {
IWritableMemChunkGroup memChunkGroup =
createAlignedMemChunkGroupIfNotExistAndGet(deviceId, schemaList);
if (memChunkGroup.writeWithFlushCheck(insertTime, objectValue, schemaList)) {
- shouldFlush = true;
+ reachChunkSizeOrPointNumThreshold = true;
}
}
@@ -353,7 +354,7 @@ public abstract class AbstractMemTable implements IMemTable {
schemaList,
start,
end)) {
- shouldFlush = true;
+ reachChunkSizeOrPointNumThreshold = true;
}
}
@@ -379,7 +380,7 @@ public abstract class AbstractMemTable implements IMemTable {
schemaList,
start,
end)) {
- shouldFlush = true;
+ reachChunkSizeOrPointNumThreshold = true;
}
}
@@ -426,11 +427,8 @@ public abstract class AbstractMemTable implements IMemTable {
}
@Override
- public boolean reachTotalPointNumThreshold() {
- if (totalPointsNum == 0) {
- return false;
- }
- return totalPointsNum >= totalPointsNumThreshold;
+ public boolean reachChunkSizeOrPointNumThreshold() {
+ return reachChunkSizeOrPointNumThreshold;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 489520b201c..934b616e393 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -87,22 +87,22 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
@Override
- public void putLong(long t, long v) {
+ public boolean putLongWithFlushCheck(long t, long v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@Override
- public void putInt(long t, int v) {
+ public boolean putIntWithFlushCheck(long t, int v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@Override
- public void putFloat(long t, float v) {
+ public boolean putFloatWithFlushCheck(long t, float v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@Override
- public void putDouble(long t, double v) {
+ public boolean putDoubleWithFlushCheck(long t, double v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@@ -112,33 +112,33 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
@Override
- public void putBoolean(long t, boolean v) {
+ public boolean putBooleanWithFlushCheck(long t, boolean v) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@Override
public boolean putAlignedValueWithFlushCheck(long t, Object[] v) {
list.putAlignedValue(t, v);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end) {
+ public boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@Override
- public void putInts(long[] t, int[] v, BitMap bitMap, int start, int end) {
+ public boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@Override
- public void putFloats(long[] t, float[] v, BitMap bitMap, int start, int end) {
+ public boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap, int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@Override
- public void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int end) {
+ public boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap, int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@@ -149,7 +149,8 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
@Override
- public void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end) {
+ public boolean putBooleansWithFlushCheck(
+ long[] t, boolean[] v, BitMap bitMap, int start, int end) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + TSDataType.VECTOR);
}
@@ -157,7 +158,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
public boolean putAlignedValuesWithFlushCheck(
long[] t, Object[] v, BitMap[] bitMaps, int start, int end) {
list.putAlignedValues(t, v, bitMaps, start, end);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
index b2288f174f8..222b4199105 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
@@ -75,12 +75,7 @@ public interface IMemTable extends WALEntryValue {
/** only used when mem control enabled */
long getTVListsRamCost();
- /**
- * only used when mem control enabled
- *
- * @return whether the average number of points in each WritableChunk reaches the threshold
- */
- boolean reachTotalPointNumThreshold();
+ boolean reachChunkSizeOrPointNumThreshold();
int getSeriesNumber();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
index 4c07c3ece33..fea1770fdee 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
@@ -30,31 +30,31 @@ import java.util.List;
public interface IWritableMemChunk extends WALEntryValue {
- void putLong(long t, long v);
+ boolean putLongWithFlushCheck(long t, long v);
- void putInt(long t, int v);
+ boolean putIntWithFlushCheck(long t, int v);
- void putFloat(long t, float v);
+ boolean putFloatWithFlushCheck(long t, float v);
- void putDouble(long t, double v);
+ boolean putDoubleWithFlushCheck(long t, double v);
boolean putBinaryWithFlushCheck(long t, Binary v);
- void putBoolean(long t, boolean v);
+ boolean putBooleanWithFlushCheck(long t, boolean v);
boolean putAlignedValueWithFlushCheck(long t, Object[] v);
- void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end);
+ boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int start, int end);
- void putInts(long[] t, int[] v, BitMap bitMap, int start, int end);
+ boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int start, int end);
- void putFloats(long[] t, float[] v, BitMap bitMap, int start, int end);
+ boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap, int start, int end);
- void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int end);
+ boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap, int start, int end);
boolean putBinariesWithFlushCheck(long[] t, Binary[] v, BitMap bitMap, int start, int end);
- void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end);
+ boolean putBooleansWithFlushCheck(long[] t, boolean[] v, BitMap bitMap, int start, int end);
boolean putAlignedValuesWithFlushCheck(
long[] t, Object[] v, BitMap[] bitMaps, int start, int end);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 680f30389b5..6233b426b3d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -43,7 +43,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNo
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
-import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.DataRegionInfo;
import org.apache.iotdb.db.storageengine.dataregion.flush.CloseFileListener;
@@ -91,6 +90,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -909,7 +909,22 @@ public class TsFileProcessor {
if (dataRegionInfo.needToReportToSystem()) {
try {
if (!SystemInfo.getInstance().reportStorageGroupStatus(dataRegionInfo, this)) {
- StorageEngine.blockInsertionIfReject(this);
+ long startTime = System.currentTimeMillis();
+ while (SystemInfo.getInstance().isRejected()) {
+ if (workMemTable.shouldFlush()) {
+ break;
+ }
+ try {
+ TimeUnit.MILLISECONDS.sleep(config.getCheckPeriodWhenInsertBlocked());
+ if (System.currentTimeMillis() - startTime
+ > config.getMaxWaitingTimeWhenInsertBlocked()) {
+ throw new WriteProcessRejectException(
+ "System rejected over " + (System.currentTimeMillis() - startTime) + "ms");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
}
} catch (WriteProcessRejectException e) {
dataRegionInfo.releaseStorageGroupMemCost(memTableIncrement);
@@ -987,13 +1002,8 @@ public class TsFileProcessor {
if (workMemTable.shouldFlush()) {
return true;
}
- if (workMemTable.reachTotalPointNumThreshold()) {
- logger.info(
- "The avg series points num {} of tsfile {} reaches the threshold",
- workMemTable.getTotalPointsNum() / workMemTable.getSeriesNumber(),
- tsFileResource.getTsFile().getAbsolutePath());
- WritingMetrics.getInstance()
- .recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
+ if (workMemTable.reachChunkSizeOrPointNumThreshold()) {
+ WritingMetrics.getInstance().recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
return true;
}
return false;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index c1a71d32a53..134bf4310e9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -61,26 +61,20 @@ public class WritableMemChunk implements IWritableMemChunk {
public boolean writeWithFlushCheck(long insertTime, Object objectValue) {
switch (schema.getType()) {
case BOOLEAN:
- putBoolean(insertTime, (boolean) objectValue);
- break;
+ return putBooleanWithFlushCheck(insertTime, (boolean) objectValue);
case INT32:
- putInt(insertTime, (int) objectValue);
- break;
+ return putIntWithFlushCheck(insertTime, (int) objectValue);
case INT64:
- putLong(insertTime, (long) objectValue);
- break;
+ return putLongWithFlushCheck(insertTime, (long) objectValue);
case FLOAT:
- putFloat(insertTime, (float) objectValue);
- break;
+ return putFloatWithFlushCheck(insertTime, (float) objectValue);
case DOUBLE:
- putDouble(insertTime, (double) objectValue);
- break;
+ return putDoubleWithFlushCheck(insertTime, (double) objectValue);
case TEXT:
return putBinaryWithFlushCheck(insertTime, (Binary) objectValue);
default:
- throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + schema.getType());
+ throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + schema.getType().name());
}
- return false;
}
@Override
@@ -95,31 +89,25 @@ public class WritableMemChunk implements IWritableMemChunk {
switch (dataType) {
case BOOLEAN:
boolean[] boolValues = (boolean[]) valueList;
- putBooleans(times, boolValues, bitMap, start, end);
- break;
+ return putBooleansWithFlushCheck(times, boolValues, bitMap, start, end);
case INT32:
int[] intValues = (int[]) valueList;
- putInts(times, intValues, bitMap, start, end);
- break;
+ return putIntsWithFlushCheck(times, intValues, bitMap, start, end);
case INT64:
long[] longValues = (long[]) valueList;
- putLongs(times, longValues, bitMap, start, end);
- break;
+ return putLongsWithFlushCheck(times, longValues, bitMap, start, end);
case FLOAT:
float[] floatValues = (float[]) valueList;
- putFloats(times, floatValues, bitMap, start, end);
- break;
+ return putFloatsWithFlushCheck(times, floatValues, bitMap, start, end);
case DOUBLE:
double[] doubleValues = (double[]) valueList;
- putDoubles(times, doubleValues, bitMap, start, end);
- break;
+ return putDoublesWithFlushCheck(times, doubleValues, bitMap, start, end);
case TEXT:
Binary[] binaryValues = (Binary[]) valueList;
return putBinariesWithFlushCheck(times, binaryValues, bitMap, start, end);
default:
- throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType);
+ throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + dataType.name());
}
- return false;
}
@Override
@@ -134,34 +122,39 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public void putLong(long t, long v) {
+ public boolean putLongWithFlushCheck(long t, long v) {
list.putLong(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putInt(long t, int v) {
+ public boolean putIntWithFlushCheck(long t, int v) {
list.putInt(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putFloat(long t, float v) {
+ public boolean putFloatWithFlushCheck(long t, float v) {
list.putFloat(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putDouble(long t, double v) {
+ public boolean putDoubleWithFlushCheck(long t, double v) {
list.putDouble(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
public boolean putBinaryWithFlushCheck(long t, Binary v) {
list.putBinary(t, v);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putBoolean(long t, boolean v) {
+ public boolean putBooleanWithFlushCheck(long t, boolean v) {
list.putBoolean(t, v);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
@@ -170,35 +163,41 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public void putLongs(long[] t, long[] v, BitMap bitMap, int start, int end) {
+ public boolean putLongsWithFlushCheck(long[] t, long[] v, BitMap bitMap, int start, int end) {
list.putLongs(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putInts(long[] t, int[] v, BitMap bitMap, int start, int end) {
+ public boolean putIntsWithFlushCheck(long[] t, int[] v, BitMap bitMap, int start, int end) {
list.putInts(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putFloats(long[] t, float[] v, BitMap bitMap, int start, int end) {
+ public boolean putFloatsWithFlushCheck(long[] t, float[] v, BitMap bitMap, int start, int end) {
list.putFloats(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putDoubles(long[] t, double[] v, BitMap bitMap, int start, int end) {
+ public boolean putDoublesWithFlushCheck(long[] t, double[] v, BitMap bitMap, int start, int end) {
list.putDoubles(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
public boolean putBinariesWithFlushCheck(
long[] t, Binary[] v, BitMap bitMap, int start, int end) {
list.putBinaries(t, v, bitMap, start, end);
- return list.reachMaxChunkSizeThreshold();
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
- public void putBooleans(long[] t, boolean[] v, BitMap bitMap, int start, int end) {
+ public boolean putBooleansWithFlushCheck(
+ long[] t, boolean[] v, BitMap bitMap, int start, int end) {
list.putBooleans(t, v, bitMap, start, end);
+ return list.reachChunkSizeOrPointNumThreshold();
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 4092db7d442..4949044d590 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -698,8 +698,8 @@ public abstract class AlignedTVList extends TVList {
}
@Override
- public boolean reachMaxChunkSizeThreshold() {
- return reachMaxChunkSizeFlag;
+ public boolean reachChunkSizeOrPointNumThreshold() {
+ return reachMaxChunkSizeFlag || rowCount >= MAX_SERIES_POINT_NUMBER;
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
index 4c5d4061530..bce5dc083c3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java
@@ -98,8 +98,8 @@ public abstract class BinaryTVList extends TVList {
}
@Override
- public boolean reachMaxChunkSizeThreshold() {
- return memoryBinaryChunkSize >= TARGET_CHUNK_SIZE;
+ public boolean reachChunkSizeOrPointNumThreshold() {
+ return memoryBinaryChunkSize >= TARGET_CHUNK_SIZE || rowCount >= MAX_SERIES_POINT_NUMBER;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index f2b26158a5f..f6c34ba0d62 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -49,6 +49,8 @@ public abstract class TVList implements WALEntryValue {
protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not consistent";
protected static final long TARGET_CHUNK_SIZE =
IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+ protected static final long MAX_SERIES_POINT_NUMBER =
+ IoTDBDescriptor.getInstance().getConfig().getAvgSeriesPointNumberThreshold();
// list of timestamp array, add 1 when expanded -> data point timestamp array
// index relation: arrayIndex -> elementIndex
protected List<long[]> timestamps;
@@ -148,8 +150,8 @@ public abstract class TVList implements WALEntryValue {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
- public boolean reachMaxChunkSizeThreshold() {
- return false;
+ public boolean reachChunkSizeOrPointNumThreshold() {
+ return rowCount >= MAX_SERIES_POINT_NUMBER;
}
public void putBoolean(long time, boolean value) {
(iotdb) 01/06: Remove usage of CompressionRatio in compaction (#12875)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 36a18f127a4caf2260d0143ddcf756573b49b775
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Mon Jul 8 22:12:18 2024 +0800
Remove usage of CompressionRatio in compaction (#12875)
* add log
* add default value
* remove empty line
* set old file name
* remove usage of CompressionRatio in compaction
* modify calculatingDataMemoryCost
* fix ut
---
.../estimator/AbstractCompactionEstimator.java | 6 +++---
.../FastCompactionInnerCompactionEstimator.java | 15 +++++++--------
.../estimator/FastCrossSpaceCompactionEstimator.java | 15 +++++++--------
.../estimator/ReadChunkInnerCompactionEstimator.java | 19 +++++++++----------
.../RepairUnsortedFileCompactionEstimator.java | 11 ++++-------
.../dataregion/flush/CompressionRatio.java | 7 +++++++
...paceCompactionWithFastPerformerValidationTest.java | 4 ++++
...ompactionWithReadPointPerformerValidationTest.java | 4 ++++
8 files changed, 45 insertions(+), 36 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
index e72f163a21c..6c28e56a5c1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
@@ -21,12 +21,13 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimat
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.IDeviceID;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -58,8 +59,7 @@ public abstract class AbstractCompactionEstimator {
protected Map<TsFileResource, DeviceTimeIndex> deviceTimeIndexCache = new HashMap<>();
protected IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
-
- protected long compressionRatio = (long) CompressionRatio.getInstance().getRatio() + 1;
+ protected TSFileConfig tsFileConfig = TSFileDescriptor.getInstance().getConfig();
protected abstract long calculatingMetadataMemoryCost(CompactionTaskInfo taskInfo);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
index 8be9b210a52..8530219671e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
@@ -55,21 +55,20 @@ public class FastCompactionInnerCompactionEstimator extends AbstractInnerSpaceEs
}
long maxConcurrentSeriesNum =
Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum());
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSizeOfTotalFiles =
- averageUncompressedChunkSize
- * taskInfo.getFileInfoList().size()
- * maxConcurrentSeriesNum
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
+ averageChunkSize
+ * taskInfo.getFileInfoList().size()
+ * maxConcurrentSeriesNum
+ * taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
long targetChunkWriterSize =
Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize);
long maxConcurrentChunkSizeFromSourceFile =
- averageUncompressedChunkSize
+ (averageChunkSize + tsFileConfig.getPageSizeInByte())
* maxConcurrentSeriesNum
* calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
index 1ac3b5d5be1..4d9b89fda52 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
@@ -56,21 +56,20 @@ public class FastCrossSpaceCompactionEstimator extends AbstractCrossSpaceEstimat
long maxConcurrentSeriesNum =
Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum());
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSizeOfTotalFiles =
- averageUncompressedChunkSize
- * taskInfo.getFileInfoList().size()
- * maxConcurrentSeriesNum
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
+ averageChunkSize
+ * taskInfo.getFileInfoList().size()
+ * maxConcurrentSeriesNum
+ * taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
long targetChunkWriterSize =
Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize);
long maxConcurrentChunkSizeFromSourceFile =
- averageUncompressedChunkSize
+ (averageChunkSize + tsFileConfig.getPageSizeInByte())
* maxConcurrentSeriesNum
* calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
index e432b1eac5d..e77a87f4c58 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
@@ -51,22 +51,21 @@ public class ReadChunkInnerCompactionEstimator extends AbstractInnerSpaceEstimat
if (taskInfo.getTotalChunkNum() == 0) {
return taskInfo.getModificationFileSize();
}
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+ long maxConcurrentSeriesNum = taskInfo.getMaxConcurrentSeriesNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSizeOfTotalFiles =
- averageUncompressedChunkSize
- * taskInfo.getFileInfoList().size()
- * taskInfo.getMaxConcurrentSeriesNum()
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
- long maxTargetChunkWriterSize =
- config.getTargetChunkSize() * taskInfo.getMaxConcurrentSeriesNum();
+ averageChunkSize
+ * taskInfo.getFileInfoList().size()
+ * maxConcurrentSeriesNum
+ * taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
+ long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
long targetChunkWriterSize =
Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize);
long chunkSizeFromSourceFile =
- averageUncompressedChunkSize * taskInfo.getMaxConcurrentSeriesNum();
+ (averageChunkSize + tsFileConfig.getPageSizeInByte()) * maxConcurrentSeriesNum;
return targetChunkWriterSize + chunkSizeFromSourceFile + taskInfo.getModificationFileSize();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
index 3eadd41ea28..c7852753822 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
@@ -50,19 +50,16 @@ public class RepairUnsortedFileCompactionEstimator extends AbstractInnerSpaceEst
}
long maxConcurrentSeriesNum =
Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum());
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSize =
- averageUncompressedChunkSize
- * maxConcurrentSeriesNum
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
+ averageChunkSize * maxConcurrentSeriesNum * taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
long targetChunkWriterSize = Math.min(maxConcurrentSeriesSize, maxTargetChunkWriterSize);
long inMemorySortedDataSize =
- averageUncompressedChunkSize
+ (averageChunkSize + tsFileConfig.getPageSizeInByte())
* Math.min(
taskInfo.getMaxChunkMetadataNumInDevice(),
taskInfo.getMaxChunkMetadataNumInSeries() * maxConcurrentSeriesNum);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
index 5d6c40e1487..2f1e173f249 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
@@ -85,6 +85,12 @@ public class CompressionRatio {
* call this method.
*/
public synchronized void updateRatio(long memorySize, long diskSize) throws IOException {
+ if (memorySize < 0 || totalMemorySize.get() < 0) {
+ LOGGER.warn(
+ "The compression ratio is negative, current memTableSize: {}, totalMemTableSize: {}",
+ memorySize,
+ totalMemorySize);
+ }
File oldFile = SystemFileFactory.INSTANCE.getFile(directory, oldFileName);
totalMemorySize.addAndGet(memorySize);
@@ -149,6 +155,7 @@ public class CompressionRatio {
"After restoring from compression ratio file, total memory size = {}, total disk size = {}",
totalMemorySize,
totalDiskSize);
+ oldFileName = ratioFiles[maxRatioIndex].getName();
deleteRedundantFilesByIndex(ratioFiles, maxRatioIndex);
} else { // If there is no new file, try to restore from the old version file
File[] ratioFilesBeforeV121 =
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index d1bf1564d2a..cf9358340f6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -79,6 +80,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
private final String oldThreadName = Thread.currentThread().getName();
private ICrossCompactionPerformer performer = new FastCompactionPerformer(true);
+ private long compactionMemory = SystemInfo.getInstance().getMemorySizeForCompaction();
@Before
public void setUp()
@@ -87,6 +89,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+ SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024);
Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
}
@@ -95,6 +98,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
super.tearDown();
Thread.currentThread().setName(oldThreadName);
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+ SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory);
TsFileValidationTool.badFileNum = 0;
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
index c1090e39450..5c0edd7bc24 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
@@ -44,6 +44,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionF
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -77,6 +78,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
private final String oldThreadName = Thread.currentThread().getName();
private ICrossCompactionPerformer performer = new ReadPointCompactionPerformer();
+ private long compactionMemory = SystemInfo.getInstance().getMemorySizeForCompaction();
@Before
public void setUp()
@@ -85,6 +87,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+ SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024);
Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
}
@@ -93,6 +96,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
super.tearDown();
Thread.currentThread().setName(oldThreadName);
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+ SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory);
TsFileValidationTool.badFileNum = 0;
}
(iotdb) 02/06: Fix negative compression ratios after restart (#12879)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5422add39882b1741b11b287c5404569e4a98a8c
Author: Haonan <hh...@outlook.com>
AuthorDate: Mon Jul 8 22:40:10 2024 +0800
Fix negative compression ratios after restart (#12879)
---
.../recover/file/UnsealedTsFileRecoverPerformer.java | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
index 080867b8800..705027ab7d5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.pipe.agent.PipeAgent;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
+import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
import org.apache.iotdb.db.storageengine.dataregion.flush.MemTableFlushTask;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunk;
@@ -259,6 +260,24 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform
// set recover progress index for pipe
PipeAgent.runtime().assignProgressIndexForTsFileRecovery(tsFileResource);
+ try {
+ long memTableSize = recoveryMemTable.memSize();
+ double compressionRatio = ((double) memTableSize) / writer.getPos();
+ logger.info(
+ "The compression ratio of tsfile {} is {}, totalMemTableSize: {}, the file size: {}",
+ writer.getFile().getAbsolutePath(),
+ String.format("%.2f", compressionRatio),
+ memTableSize,
+ writer.getPos());
+ CompressionRatio.getInstance().updateRatio(memTableSize, writer.getPos());
+ } catch (IOException e) {
+ logger.error(
+ "{}: {} update compression ratio failed",
+ databaseName,
+ tsFileResource.getTsFile().getName(),
+ e);
+ }
+
// if we put following codes in the 'if' clause above, this file can be continued writing
// into it
// currently, we close this file anyway
(iotdb) 03/06: modify compaction schedule (#12826)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5f9291c9746dfaaf207cd9b09b1b7a2bd417b1dc
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Thu Jul 4 13:10:34 2024 +0800
modify compaction schedule (#12826)
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../compaction/schedule/CompactionScheduler.java | 20 +++++++++++++++-----
.../compaction/schedule/CompactionTaskManager.java | 8 ++++++++
3 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 1dd9c0c3942..6c870287b56 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -470,7 +470,7 @@ public class IoTDBConfig {
* cross space compaction, eliminate the unsequence files first BALANCE: alternate two compaction
* types
*/
- private CompactionPriority compactionPriority = CompactionPriority.BALANCE;
+ private CompactionPriority compactionPriority = CompactionPriority.INNER_CROSS;
private double chunkMetadataSizeProportion = 0.1;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
index 8d57caee8fd..097caad018a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
@@ -100,11 +100,18 @@ public class CompactionScheduler {
// the name of this variable is trySubmitCount, because the task submitted to the queue could be
// evicted due to the low priority of the task
int trySubmitCount = 0;
- trySubmitCount += tryToSubmitCrossSpaceCompactionTask(tsFileManager, timePartition, summary);
- trySubmitCount +=
- tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, true, summary);
- trySubmitCount +=
- tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, false, summary);
+ try {
+ trySubmitCount +=
+ tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, true, summary);
+ trySubmitCount +=
+ tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, false, summary);
+ trySubmitCount += tryToSubmitCrossSpaceCompactionTask(tsFileManager, timePartition, summary);
+ } catch (InterruptedException e) {
+ throw e;
+ } catch (Throwable e) {
+ LOGGER.error("Meet error in compaction schedule.", e);
+ }
+
return trySubmitCount;
}
@@ -248,6 +255,9 @@ public class CompactionScheduler {
if (!config.isEnableCrossSpaceCompaction()) {
return 0;
}
+ if (!CompactionTaskManager.getInstance().shouldSelectCrossSpaceCompactionTask()) {
+ return 0;
+ }
String logicalStorageGroupName = tsFileManager.getStorageGroupName();
String dataRegionId = tsFileManager.getDataRegionId();
long compactionConfigVersionWhenSelectTask =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
index 78117ae9d41..a1d98016fd2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
@@ -243,6 +243,14 @@ public class CompactionTaskManager implements IService {
return ServiceType.COMPACTION_SERVICE;
}
+ public boolean shouldSelectCrossSpaceCompactionTask() {
+ // If the queue size accounts for less than 0.8 of the total capacity, select cross space
+ // compaction task
+ int waitingQueueRestSize =
+ candidateCompactionTaskQueue.getMaxSize() - candidateCompactionTaskQueue.size();
+ return 5 * waitingQueueRestSize >= candidateCompactionTaskQueue.size();
+ }
+
public boolean isWaitingQueueFull() {
return candidateCompactionTaskQueue.size() == candidateCompactionTaskQueue.getMaxSize();
}
(iotdb) 06/06: CI
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cb524977371d17e78980be960f68a5ebdc86dbb8
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Jul 11 15:48:21 2024 +0800
CI
---
.github/workflows/cluster-it-1c1d.yml | 2 +-
.github/workflows/cluster-it-1c3d.yml | 2 +-
.github/workflows/pipe-it-2cluster.yml | 2 +-
.github/workflows/unit-test.yml | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml
index e897770dabf..edf54ebcba3 100644
--- a/.github/workflows/cluster-it-1c1d.yml
+++ b/.github/workflows/cluster-it-1c1d.yml
@@ -12,7 +12,7 @@ on:
pull_request:
branches:
- master
- - 'rel/1.*'
+ - 'rc/*'
- pipe-meta-sync
paths-ignore:
- 'docs/**'
diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml
index c2edca6b9d2..2091aaeb5c3 100644
--- a/.github/workflows/cluster-it-1c3d.yml
+++ b/.github/workflows/cluster-it-1c3d.yml
@@ -12,7 +12,7 @@ on:
pull_request:
branches:
- master
- - 'rel/1.*'
+ - 'rc/*'
- pipe-meta-sync
paths-ignore:
- 'docs/**'
diff --git a/.github/workflows/pipe-it-2cluster.yml b/.github/workflows/pipe-it-2cluster.yml
index 3e2313a490d..619993accb3 100644
--- a/.github/workflows/pipe-it-2cluster.yml
+++ b/.github/workflows/pipe-it-2cluster.yml
@@ -11,7 +11,7 @@ on:
pull_request:
branches:
- master
- - 'rel/1.*'
+ - 'rc/*'
paths-ignore:
- 'docs/**'
- 'site/**'
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 72594a79aea..c780cea8a15 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -15,7 +15,7 @@ on:
pull_request:
branches:
- master
- - 'rel/*'
+ - 'rc/*'
- pipe-meta-sync
paths-ignore:
- 'docs/**'
(iotdb) 05/06: spotless
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5167cb95bd156866bd7b7e6f8d2713ccdcaee5ed
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Jul 11 15:47:32 2024 +0800
spotless
---
.../iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 6233b426b3d..b140b840cdf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -1003,7 +1003,8 @@ public class TsFileProcessor {
return true;
}
if (workMemTable.reachChunkSizeOrPointNumThreshold()) {
- WritingMetrics.getInstance().recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
+ WritingMetrics.getInstance()
+ .recordSeriesFullFlushMemTableCount(dataRegionInfo.getDataRegion().getDataRegionId(), 1);
return true;
}
return false;