You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sp...@apache.org on 2023/02/27 15:30:34 UTC
[iotdb] branch master updated: [IOTDB-5587] Add dataRegionId into Memtable and update flush point metrics (#9148)
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9cd09d903f [IOTDB-5587] Add dataRegionId into Memtable and update flush point metrics (#9148)
9cd09d903f is described below
commit 9cd09d903fb0737042c5f37be4374baee61c5575
Author: ZhangHongYin <46...@users.noreply.github.com>
AuthorDate: Mon Feb 27 23:30:26 2023 +0800
[IOTDB-5587] Add dataRegionId into Memtable and update flush point metrics (#9148)
* Add dataRegionId into Memtable and update flush
* Fix Bug
* Fix format
---
.../apache/iotdb/db/engine/flush/MemTableFlushTask.java | 17 ++++++++++-------
.../iotdb/db/engine/storagegroup/TsFileProcessor.java | 6 +++++-
.../recover/file/UnsealedTsFileRecoverPerformer.java | 5 ++++-
.../iotdb/db/engine/memtable/MemTableFlushTaskTest.java | 10 +++++++---
4 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index 138af6962e..a0400d0b33 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup;
import org.apache.iotdb.db.exception.runtime.FlushRunTimeException;
import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
import org.apache.iotdb.db.rescon.SystemInfo;
-import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.utils.IoTDBMetricsUtils;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
@@ -70,6 +69,7 @@ public class MemTableFlushTask {
: new LinkedBlockingQueue<>();
private String storageGroup;
+ private String dataRegionId;
private IMemTable memTable;
@@ -82,10 +82,14 @@ public class MemTableFlushTask {
* @param storageGroup current database
*/
public MemTableFlushTask(
- IMemTable memTable, RestorableTsFileIOWriter writer, String storageGroup) {
+ IMemTable memTable,
+ RestorableTsFileIOWriter writer,
+ String storageGroup,
+ String dataRegionId) {
this.memTable = memTable;
this.writer = writer;
this.storageGroup = storageGroup;
+ this.dataRegionId = dataRegionId;
this.encodingTaskFuture = SUB_TASK_POOL_MANAGER.submit(encodingTask);
this.ioTaskFuture = SUB_TASK_POOL_MANAGER.submit(ioTask);
LOGGER.debug(
@@ -253,10 +257,7 @@ public class MemTableFlushTask {
Thread.currentThread().interrupt();
}
- if (!storageGroup.startsWith(IoTDBMetricsUtils.DATABASE)
- && MetricLevel.higherOrEqual(
- MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel(),
- MetricLevel.CORE)) {
+ if (!storageGroup.startsWith(IoTDBMetricsUtils.DATABASE)) {
int lastIndex = storageGroup.lastIndexOf("-");
if (lastIndex == -1) {
lastIndex = storageGroup.length();
@@ -269,7 +270,9 @@ public class MemTableFlushTask {
Tag.DATABASE.toString(),
storageGroup.substring(0, lastIndex),
Tag.TYPE.toString(),
- "flush");
+ "flush",
+ Tag.REGION.toString(),
+ dataRegionId);
}
LOGGER.info(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 054a975102..a7dace8019 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -1076,7 +1076,11 @@ public class TsFileProcessor {
try {
writer.mark();
MemTableFlushTask flushTask =
- new MemTableFlushTask(memTableToFlush, writer, storageGroupName);
+ new MemTableFlushTask(
+ memTableToFlush,
+ writer,
+ storageGroupName,
+ dataRegionInfo.getDataRegion().getDataRegionId());
flushTask.syncFlushMemTable();
} catch (Throwable e) {
if (writer == null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java
index 0fd8940839..d7629d79cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java
@@ -233,7 +233,10 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform
tsFileResource.getTsFile().getParentFile().getParentFile().getParentFile().getName();
MemTableFlushTask tableFlushTask =
new MemTableFlushTask(
- recoveryMemTable, writer, databaseName + FILE_NAME_SEPARATOR + dataRegionId);
+ recoveryMemTable,
+ writer,
+ databaseName + FILE_NAME_SEPARATOR + dataRegionId,
+ dataRegionId);
tableFlushTask.syncFlushMemTable();
tsFileResource.updatePlanIndexes(recoveryMemTable.getMinPlanIndex());
tsFileResource.updatePlanIndexes(recoveryMemTable.getMaxPlanIndex());
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
index 13e92e33a8..e94a145898 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java
@@ -41,6 +41,7 @@ public class MemTableFlushTaskTest {
private RestorableTsFileIOWriter writer;
private String storageGroup = "storage_group1";
+ private String dataRegionId = "DataRegion-1";
private String filePath =
TestConstant.OUTPUT_DATA_DIR.concat("testUnsealedTsFileProcessor.tsfile");
private IMemTable memTable;
@@ -71,7 +72,8 @@ public class MemTableFlushTaskTest {
MemTableTestUtils.deviceId0,
MemTableTestUtils.measurementId0,
MemTableTestUtils.dataType0);
- MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, writer, storageGroup);
+ MemTableFlushTask memTableFlushTask =
+ new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId);
assertTrue(
writer
.getVisibleMetadataList(
@@ -107,7 +109,8 @@ public class MemTableFlushTaskTest {
public void testFlushVectorMemTable()
throws ExecutionException, InterruptedException, IllegalPathException, WriteProcessException {
MemTableTestUtils.produceVectorData(memTable);
- MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, writer, storageGroup);
+ MemTableFlushTask memTableFlushTask =
+ new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId);
assertTrue(
writer
.getVisibleMetadataList(MemTableTestUtils.deviceId0, "sensor0", TSDataType.BOOLEAN)
@@ -134,7 +137,8 @@ public class MemTableFlushTaskTest {
public void testFlushNullableVectorMemTable()
throws ExecutionException, InterruptedException, IllegalPathException, WriteProcessException {
MemTableTestUtils.produceNullableVectorData(memTable);
- MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, writer, storageGroup);
+ MemTableFlushTask memTableFlushTask =
+ new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId);
assertTrue(
writer
.getVisibleMetadataList(MemTableTestUtils.deviceId0, "sensor0", TSDataType.BOOLEAN)