You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/04/29 07:05:43 UTC
(iotdb) branch drop_database_avoid_flush created (now 2865f2e6b60)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch drop_database_avoid_flush
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 2865f2e6b60 Add more UT
This branch includes the following new commits:
new 01bc1957237 Avoid flush when drop database
new 2865f2e6b60 Add more UT
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/02: Avoid flush when drop database
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch drop_database_avoid_flush
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 01bc19572373f85f7e6eca04f53dd261ba660bd8
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Apr 29 10:34:29 2024 +0800
Avoid flush when drop database
---
.../org/apache/iotdb/db/storageengine/dataregion/DataRegion.java | 7 +++----
.../db/storageengine/dataregion/memtable/TsFileProcessor.java | 2 ++
.../apache/iotdb/db/storageengine/dataregion/DataRegionTest.java | 5 +++--
.../dataregion/compaction/settle/SettleRequestHandlerTest.java | 3 ++-
4 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 6c62a289902..0d32a95f974 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1548,13 +1548,12 @@ public class DataRegion implements IDataRegionForQuery {
}
/** delete tsfile */
- public void syncDeleteDataFiles() {
+ public void syncDeleteDataFiles() throws TsFileProcessorException {
logger.info(
"{} will close all files for deleting data files", databaseName + "-" + dataRegionId);
writeLock("syncDeleteDataFiles");
try {
-
- syncCloseAllWorkingTsFileProcessors();
+ forceCloseAllWorkingTsFileProcessors();
// normally, mergingModification is just need to be closed by after a merge task is finished.
// we close it here just for IT test.
closeAllResources();
@@ -1735,7 +1734,7 @@ public class DataRegion implements IDataRegionForQuery {
}
/** close all working tsfile processors */
- public List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
+ List<Future<?>> asyncCloseAllWorkingTsFileProcessors() {
writeLock("asyncCloseAllWorkingTsFileProcessors");
List<Future<?>> futures = new ArrayList<>();
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 57944b67ab9..0cbc2ea3768 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -1498,6 +1498,8 @@ public class TsFileProcessor {
} catch (IOException e) {
throw new TsFileProcessorException(e);
}
+ tsFileProcessorInfo.clear();
+ dataRegionInfo.closeTsFileProcessorAndReportToSystem(this);
}
public void setTsFileProcessorInfo(TsFileProcessorInfo tsFileProcessorInfo) {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index aa2e0a5e37c..655ee6563c3 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.DataRegionException;
+import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
@@ -866,7 +867,7 @@ public class DataRegionTest {
@Test
public void testInsertUnSequenceRows()
throws IllegalPathException, WriteProcessRejectException, QueryProcessException,
- DataRegionException {
+ DataRegionException, TsFileProcessorException {
int defaultAvgSeriesPointNumberThreshold = config.getAvgSeriesPointNumberThreshold();
config.setAvgSeriesPointNumberThreshold(2);
DataRegion dataRegion1 = new DummyDataRegion(systemDir, "root.Rows");
@@ -901,7 +902,7 @@ public class DataRegionTest {
@Test
public void testSmallReportProportionInsertRow()
throws WriteProcessException, QueryProcessException, IllegalPathException, IOException,
- DataRegionException {
+ DataRegionException, TsFileProcessorException {
double defaultValue = config.getWriteMemoryVariationReportProportion();
config.setWriteMemoryVariationReportProportion(0);
DataRegion dataRegion1 = new DummyDataRegion(systemDir, "root.ln22");
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java
index 88f7bec5ea0..176ecd07840 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandlerTest.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.DataRegionException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
@@ -79,7 +80,7 @@ public class SettleRequestHandlerTest {
}
@After
- public void tearDown() throws StorageEngineException, IOException {
+ public void tearDown() throws StorageEngineException, IOException, TsFileProcessorException {
WALManager.getInstance().stop();
FlushManager.getInstance().stop();
if (dataRegion != null) {
(iotdb) 02/02: Add more UT
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch drop_database_avoid_flush
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2865f2e6b604a2041b9b998610932c77aff61b67
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Apr 29 15:05:23 2024 +0800
Add more UT
---
.../db/storageengine/dataregion/DataRegion.java | 3 +
.../dataregion/memtable/TsFileProcessor.java | 1 +
.../rescon/memory/TimePartitionManager.java | 14 ++++
.../storageengine/dataregion/DataRegionTest.java | 86 ++++++++++++++++++++++
.../rescon/memory/TimePartitionManagerTest.java | 3 +
5 files changed, 107 insertions(+)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 0d32a95f974..d816fe69347 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1573,6 +1573,8 @@ public class DataRegion implements IDataRegionForQuery {
this.tsFileManager.clear();
lastFlushTimeMap.clearFlushedTime();
lastFlushTimeMap.clearGlobalFlushedTime();
+ TimePartitionManager.getInstance()
+ .removeTimePartitionInfo(new DataRegionId(Integer.parseInt(dataRegionId)));
} finally {
writeUnlock();
}
@@ -1770,6 +1772,7 @@ public class DataRegion implements IDataRegionForQuery {
new ArrayList<>(workUnsequenceTsFileProcessors.values())) {
tsFileProcessor.putMemTableBackAndClose();
}
+ WritingMetrics.getInstance().recordActiveTimePartitionCount(-1);
} finally {
writeUnlock();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 0cbc2ea3768..b39421fa1ec 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -1491,6 +1491,7 @@ public class TsFileProcessor {
public void putMemTableBackAndClose() throws TsFileProcessorException {
if (workMemTable != null) {
workMemTable.release();
+ dataRegionInfo.releaseStorageGroupMemCost(workMemTable.getTVListsRamCost());
workMemTable = null;
}
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java
index 89e68c49962..0d545af0c98 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java
@@ -136,6 +136,20 @@ public class TimePartitionManager {
}
}
+ public void removeTimePartitionInfo(DataRegionId dataRegionId) {
+ synchronized (timePartitionInfoMap) {
+ Map<Long, TimePartitionInfo> timePartitionInfoMapForRegion =
+ timePartitionInfoMap.remove(dataRegionId);
+ if (timePartitionInfoMapForRegion != null) {
+ for (TimePartitionInfo timePartitionInfo : timePartitionInfoMapForRegion.values()) {
+ if (timePartitionInfo != null) {
+ memCost -= timePartitionInfo.memSize + Long.BYTES;
+ }
+ }
+ }
+ }
+ }
+
@TestOnly
public TimePartitionInfo getTimePartitionInfo(DataRegionId dataRegionId, long timePartitionId) {
synchronized (timePartitionInfoMap) {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index 655ee6563c3..fc8e8f7968f 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -55,6 +55,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.constant.TestConstant;
@@ -94,8 +95,12 @@ public class DataRegionTest {
private DataRegion dataRegion;
private QueryContext context = EnvironmentUtils.TEST_QUERY_CONTEXT;
+ private double preWriteMemoryVariationReportProportion =
+ config.getWriteMemoryVariationReportProportion();
+
@Before
public void setUp() throws Exception {
+ config.setWriteMemoryVariationReportProportion(0);
EnvironmentUtils.envSetUp();
dataRegion = new DummyDataRegion(systemDir, storageGroup);
StorageEngine.getInstance().setDataRegion(new DataRegionId(0), dataRegion);
@@ -111,6 +116,7 @@ public class DataRegionTest {
EnvironmentUtils.cleanDir(TestConstant.OUTPUT_DATA_DIR);
CompactionTaskManager.getInstance().stop();
EnvironmentUtils.cleanEnv();
+ config.setWriteMemoryVariationReportProportion(preWriteMemoryVariationReportProportion);
}
public static InsertRowNode buildInsertRowNodeByTSRecord(TSRecord record)
@@ -310,6 +316,86 @@ public class DataRegionTest {
}
}
+ @Test
+ public void testIoTDBTabletWriteAndDeleteDataRegion()
+ throws QueryProcessException, IllegalPathException, WriteProcessException,
+ TsFileProcessorException {
+ String[] measurements = new String[2];
+ measurements[0] = "s0";
+ measurements[1] = "s1";
+ TSDataType[] dataTypes = new TSDataType[2];
+ dataTypes[0] = TSDataType.INT32;
+ dataTypes[1] = TSDataType.INT64;
+
+ MeasurementSchema[] measurementSchemas = new MeasurementSchema[2];
+ measurementSchemas[0] = new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN);
+ measurementSchemas[1] = new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN);
+
+ long[] times = new long[100];
+ Object[] columns = new Object[2];
+ columns[0] = new int[100];
+ columns[1] = new long[100];
+
+ for (int r = 0; r < 100; r++) {
+ times[r] = r;
+ ((int[]) columns[0])[r] = 1;
+ ((long[]) columns[1])[r] = 1;
+ }
+
+ InsertTabletNode insertTabletNode1 =
+ new InsertTabletNode(
+ new QueryId("test_write").genPlanNodeId(),
+ new PartialPath("root.vehicle.d0"),
+ false,
+ measurements,
+ dataTypes,
+ measurementSchemas,
+ times,
+ null,
+ columns,
+ times.length);
+
+ dataRegion.insertTablet(insertTabletNode1);
+
+ for (int r = 50; r < 149; r++) {
+ times[r - 50] = r;
+ ((int[]) columns[0])[r - 50] = 1;
+ ((long[]) columns[1])[r - 50] = 1;
+ }
+
+ InsertTabletNode insertTabletNode2 =
+ new InsertTabletNode(
+ new QueryId("test_write").genPlanNodeId(),
+ new PartialPath("root.vehicle.d0"),
+ false,
+ measurements,
+ dataTypes,
+ measurementSchemas,
+ times,
+ null,
+ columns,
+ times.length);
+
+ dataRegion.insertTablet(insertTabletNode2);
+ Assert.assertTrue(SystemInfo.getInstance().getTotalMemTableSize() > 0);
+ System.out.println(SystemInfo.getInstance().getTotalMemTableSize());
+ long start = System.nanoTime();
+ dataRegion.syncDeleteDataFiles();
+ System.out.println(System.nanoTime() - start);
+ Assert.assertEquals(0, SystemInfo.getInstance().getTotalMemTableSize());
+
+ QueryDataSource queryDataSource =
+ dataRegion.query(
+ Collections.singletonList(new PartialPath(deviceId, measurementId)),
+ deviceId,
+ context,
+ null,
+ null);
+
+ Assert.assertEquals(0, queryDataSource.getSeqResources().size());
+ Assert.assertEquals(0, queryDataSource.getUnseqResources().size());
+ }
+
@Test
public void testEmptyTabletWriteAndSyncClose()
throws QueryProcessException, IllegalPathException, WriteProcessException {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java
index 518340a942c..c8cd83b253e 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java
@@ -105,6 +105,9 @@ public class TimePartitionManagerTest {
timePartitionManager.updateAfterFlushing(new DataRegionId(0), 1L, 107L, 60L, true);
Assert.assertNull(timePartitionManager.getTimePartitionInfo(new DataRegionId(3), 0L));
+
+ timePartitionManager.removeTimePartitionInfo(new DataRegionId(4));
+ Assert.assertNull(timePartitionManager.getTimePartitionInfo(new DataRegionId(4), 0L));
}
@Test