You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2025/12/22 09:51:00 UTC
(iotdb) branch object_delet created (now b5a55494ac8)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch object_delet
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at b5a55494ac8 add some IT
This branch includes the following new commits:
new ee39ee10811 deving
new e2383e13ad0 dev
new fdbbd52a12c remove empty object dir
new 6bbf453f62f optimize drop table
new b5a55494ac8 add some IT
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/05: deving
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_delet
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ee39ee10811d43567acc33bd24977d3ae1e22ca9
Author: HTHou <ha...@apache.org>
AuthorDate: Mon Dec 15 14:17:01 2025 +0800
deving
---
.../db/storageengine/dataregion/DataRegion.java | 14 +++++++++++--
.../db/storageengine/rescon/disk/TierManager.java | 24 ++++++++++++++++++++++
2 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 09c9ee84304..639b7e4c214 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2751,8 +2751,8 @@ public class DataRegion implements IDataRegionForQuery {
if (deleted) {
return;
}
- TableDeviceSchemaCache.getInstance()
- .invalidateLastCache(getDatabaseName(), modEntries.get(0).getTableName());
+ String tableName = modEntries.get(0).getTableName();
+ TableDeviceSchemaCache.getInstance().invalidateLastCache(getDatabaseName(), tableName);
List<WALFlushListener> walListeners = logDeletionInWAL(node);
for (WALFlushListener walFlushListener : walListeners) {
@@ -2762,6 +2762,10 @@ public class DataRegion implements IDataRegionForQuery {
}
}
+ if (TierManager.getInstance().checkObjectPathExist(dataRegionIdString, tableName)) {
+ deleteObjectFiles(tableName, modEntries);
+ }
+
List<List<TsFileResource>> sealedTsFileResourceLists = new ArrayList<>(modEntries.size());
for (TableDeletionEntry modEntry : modEntries) {
List<TsFileResource> sealedTsFileResource = new ArrayList<>();
@@ -2930,6 +2934,12 @@ public class DataRegion implements IDataRegionForQuery {
return walFlushListeners;
}
+ private void deleteObjectFiles(String tableName, List<TableDeletionEntry> modEntries) {
+ for (TableDeletionEntry modEntry : modEntries) {
+
+ }
+ }
+
/**
* For IoTConsensus sync. See <a href="https://github.com/apache/iotdb/pull/12955">github pull
* request</a> for details.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index a5fa8b54e7b..e173bb0abaf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.rescon.disk.strategy.MinFolderOccupiedS
import org.apache.iotdb.db.storageengine.rescon.disk.strategy.RandomOnDiskUsableSpaceStrategy;
import org.apache.iotdb.metrics.utils.FileStoreUtils;
+import com.google.common.io.BaseEncoding;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.fileSystem.FSType;
import org.apache.tsfile.utils.FSUtils;
@@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.FileStore;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -272,6 +274,28 @@ public class TierManager {
return Optional.empty();
}
+ public boolean checkObjectPathExist(String regionIdStr, String... path) {
+ StringBuilder objectPath = new StringBuilder();
+ objectPath.append(regionIdStr);
+ for (String str : path) {
+ objectPath
+ .append(File.separator)
+ .append(
+ config.getRestrictObjectLimit()
+ ? str
+ : BaseEncoding.base32()
+ .omitPadding()
+ .encode(str.getBytes(StandardCharsets.UTF_8)));
+ }
+ for (String objectDir : objectDirs) {
+ File objectFilePath = FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString());
+ if (objectFilePath.exists()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public int getTiersNum() {
return seqTiers.size();
}
(iotdb) 02/05: dev
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_delet
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e2383e13ad0edac8e8a0f110e67a7e78a02720af
Author: HTHou <ha...@apache.org>
AuthorDate: Tue Dec 16 09:53:55 2025 +0800
dev
---
.../db/storageengine/dataregion/DataRegion.java | 65 ++++++++++++++++++++--
.../db/storageengine/rescon/disk/TierManager.java | 10 ++--
.../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 21 +++++++
3 files changed, 87 insertions(+), 9 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 639b7e4c214..27a290b60fd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -158,6 +158,7 @@ import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.db.utils.EncryptDBUtils;
import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.iotdb.db.utils.ObjectTypeUtils;
import org.apache.iotdb.db.utils.ObjectWriter;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.rpc.RpcUtils;
@@ -165,11 +166,13 @@ import org.apache.iotdb.rpc.TSStatusCode;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.common.io.BaseEncoding;
import org.apache.thrift.TException;
import org.apache.tsfile.external.commons.io.FileUtils;
import org.apache.tsfile.external.commons.lang3.tuple.Triple;
import org.apache.tsfile.file.metadata.ChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.fileSystem.FSType;
@@ -186,6 +189,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -2762,8 +2766,10 @@ public class DataRegion implements IDataRegionForQuery {
}
}
- if (TierManager.getInstance().checkObjectPathExist(dataRegionIdString, tableName)) {
- deleteObjectFiles(tableName, modEntries);
+ List<File> matchedObjectDirs =
+ TierManager.getInstance().getAllMatchedObjectDirs(dataRegionIdString, tableName);
+ if (!matchedObjectDirs.isEmpty()) {
+ deleteObjectFiles(matchedObjectDirs, modEntries);
}
List<List<TsFileResource>> sealedTsFileResourceLists = new ArrayList<>(modEntries.size());
@@ -2934,9 +2940,58 @@ public class DataRegion implements IDataRegionForQuery {
return walFlushListeners;
}
- private void deleteObjectFiles(String tableName, List<TableDeletionEntry> modEntries) {
- for (TableDeletionEntry modEntry : modEntries) {
-
+ private void deleteObjectFiles(List<File> matchedObjectDirs, List<TableDeletionEntry> modEntries)
+ throws IOException {
+ for (File matchedObjectDir : matchedObjectDirs) {
+ try (Stream<Path> paths = Files.walk(matchedObjectDir.toPath())) {
+ paths
+ .filter(Files::isRegularFile)
+ .filter(
+ path -> {
+ String name = path.getFileName().toString();
+ return name.endsWith(".bin");
+ })
+ .forEach(
+ path -> {
+ Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path);
+ String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2];
+ for (int i = 0; i < ideviceIdSegments.length; i++) {
+ ideviceIdSegments[i] =
+ config.getRestrictObjectLimit()
+ ? relativePath.getName(i).toString()
+ : new String(
+ BaseEncoding.base32()
+ .omitPadding()
+ .decode(relativePath.getName(i).toString()),
+ StandardCharsets.UTF_8);
+ }
+ IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments);
+ String measurementId =
+ config.getRestrictObjectLimit()
+ ? relativePath.getName(relativePath.getNameCount() - 2).toString()
+ : new String(
+ BaseEncoding.base32()
+ .omitPadding()
+ .decode(
+ relativePath
+ .getName(relativePath.getNameCount() - 2)
+ .toString()),
+ StandardCharsets.UTF_8);
+ String fileName = path.getFileName().toString();
+ long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.')));
+ logger.info(
+ "timestamp {}, measurementId {}, ideviceId {}",
+ timestamp,
+ measurementId,
+ iDeviceID);
+ for (TableDeletionEntry modEntry : modEntries) {
+ if (modEntry.affects(iDeviceID, timestamp, timestamp)
+ && modEntry.affects(measurementId)) {
+ ObjectTypeUtils.deleteObjectPath(path.toFile());
+ }
+ }
+ });
+ }
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index e173bb0abaf..a673e44dff9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -274,7 +274,8 @@ public class TierManager {
return Optional.empty();
}
- public boolean checkObjectPathExist(String regionIdStr, String... path) {
+ public List<File> getAllMatchedObjectDirs(String regionIdStr, String... path) {
+ List<File> matchedDirs = new ArrayList<>();
StringBuilder objectPath = new StringBuilder();
objectPath.append(regionIdStr);
for (String str : path) {
@@ -288,12 +289,13 @@ public class TierManager {
.encode(str.getBytes(StandardCharsets.UTF_8)));
}
for (String objectDir : objectDirs) {
- File objectFilePath = FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString());
+ File objectFilePath =
+ FSFactoryProducer.getFSFactory().getFile(objectDir, objectPath.toString());
if (objectFilePath.exists()) {
- return true;
+ matchedDirs.add(objectFilePath);
}
}
- return false;
+ return matchedDirs;
}
public int getTiersNum() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
index ec1fd592617..2a03facbf96 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
@@ -294,6 +294,27 @@ public class ObjectTypeUtils {
}
}
+ public static void deleteObjectPath(File file) {
+ File tmpFile = new File(file.getPath() + ".tmp");
+ File bakFile = new File(file.getPath() + ".back");
+ for (int i = 0; i < 2; i++) {
+ if (file.exists()) {
+ FileMetrics.getInstance().decreaseObjectFileNum(1);
+ FileMetrics.getInstance().decreaseObjectFileSize(file.length());
+ }
+ try {
+ deleteObjectFile(file);
+ deleteObjectFile(tmpFile);
+ deleteObjectFile(bakFile);
+ } catch (IOException e) {
+ logger.error("Failed to remove object file {}", file.getAbsolutePath(), e);
+ }
+ }
+ if (file.getParentFile().exists()) {
+
+ }
+ }
+
private static void deleteObjectFile(File file) throws IOException {
if (file.exists()) {
logger.info("Remove object file {}, size is {}(byte)", file.getAbsolutePath(), file.length());
(iotdb) 05/05: add some IT
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_delet
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b5a55494ac8be02197f5159c5af32bacb626130e
Author: HTHou <ha...@apache.org>
AuthorDate: Mon Dec 22 17:50:36 2025 +0800
add some IT
---
.../relational/it/session/IoTDBObjectDeleteIT.java | 273 +++++++++++++++++++++
.../db/storageengine/dataregion/DataRegion.java | 6 +-
2 files changed, 277 insertions(+), 2 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java
new file mode 100644
index 00000000000..8bfec578671
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.session;
+
+import org.apache.iotdb.isession.ITableSession;
+import org.apache.iotdb.isession.SessionDataSet;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+
+import com.google.common.io.BaseEncoding;
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.write.record.Tablet;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertNull;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBObjectDeleteIT {
+
+ @BeforeClass
+ public static void classSetUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1");
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1");
+ }
+ }
+
+ @AfterClass
+ public static void classTearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void deleteObjectTest()
+ throws IoTDBConnectionException, StatementExecutionException, IOException {
+ String testObject =
+ System.getProperty("user.dir")
+ + File.separator
+ + "target"
+ + File.separator
+ + "test-classes"
+ + File.separator
+ + "object-example.pt";
+
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ // insert table data by tablet
+ List<String> columnNameList =
+ Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
+ List<TSDataType> dataTypeList =
+ Arrays.asList(
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.FLOAT,
+ TSDataType.OBJECT);
+ List<ColumnCategory> columnTypeList =
+ new ArrayList<>(
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
+ int rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, 1);
+ tablet.addValue(rowIndex, 0, "1");
+ tablet.addValue(rowIndex, 1, "5");
+ tablet.addValue(rowIndex, 2, "3");
+ tablet.addValue(rowIndex, 3, 37.6F);
+ tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject)));
+ session.insert(tablet);
+ tablet.reset();
+
+ try (SessionDataSet dataSet =
+ session.executeQueryStatement(
+ "select READ_OBJECT(file) from object_table where time = 1")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ Binary binary = iterator.getBlob(1);
+ Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues());
+ }
+ session.executeNonQueryStatement("DROP TABLE IF EXISTS object_table");
+ }
+ }
+
+ // test object file path
+ boolean success = false;
+ for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
+ File objectDir = new File(objectDirStr);
+ if (objectDir.exists() && objectDir.isDirectory()) {
+ File[] regionDirs = objectDir.listFiles();
+ if (regionDirs != null) {
+ for (File regionDir : regionDirs) {
+ if (regionDir.isDirectory()) {
+ File objectFile =
+ new File(
+ regionDir,
+ convertPathString("object_table")
+ + File.separator
+ + convertPathString("1")
+ + File.separator
+ + convertPathString("5")
+ + File.separator
+ + convertPathString("3")
+ + File.separator
+ + convertPathString("file")
+ + File.separator
+ + "1.bin");
+ if (objectFile.exists() && objectFile.isFile()) {
+ success = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ Assert.assertFalse(success);
+ }
+
+ @Test
+ public void deleteObjectSegmentsTest()
+ throws IoTDBConnectionException, StatementExecutionException, IOException {
+ String testObject =
+ System.getProperty("user.dir")
+ + File.separator
+ + "target"
+ + File.separator
+ + "test-classes"
+ + File.separator
+ + "object-example.pt";
+ byte[] objectBytes = Files.readAllBytes(Paths.get(testObject));
+ List<byte[]> objectSegments = new ArrayList<>();
+ for (int i = 0; i < objectBytes.length; i += 512) {
+ objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512, objectBytes.length)));
+ }
+
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ // insert table data by tablet
+ List<String> columnNameList =
+ Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
+ List<TSDataType> dataTypeList =
+ Arrays.asList(
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.FLOAT,
+ TSDataType.OBJECT);
+ List<ColumnCategory> columnTypeList =
+ new ArrayList<>(
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
+ for (int i = 0; i < objectSegments.size() - 1; i++) {
+ int rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, 1);
+ tablet.addValue(rowIndex, 0, "1");
+ tablet.addValue(rowIndex, 1, "5");
+ tablet.addValue(rowIndex, 2, "3");
+ tablet.addValue(rowIndex, 3, 37.6F);
+ tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i));
+ session.insert(tablet);
+ tablet.reset();
+ }
+ session.executeNonQueryStatement("DELETE FROM object_table where time = 1");
+
+ try (SessionDataSet dataSet =
+ session.executeQueryStatement("select file from object_table where time = 1")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ assertNull(iterator.getString(1));
+ }
+ }
+ }
+
+ // test object file path
+ boolean success = false;
+ for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
+ String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
+ File objectDir = new File(objectDirStr);
+ if (objectDir.exists() && objectDir.isDirectory()) {
+ File[] regionDirs = objectDir.listFiles();
+ if (regionDirs != null) {
+ for (File regionDir : regionDirs) {
+ if (regionDir.isDirectory()) {
+ File objectTmpFile =
+ new File(
+ regionDir,
+ convertPathString("object_table")
+ + File.separator
+ + convertPathString("1")
+ + File.separator
+ + convertPathString("5")
+ + File.separator
+ + convertPathString("3")
+ + File.separator
+ + convertPathString("file")
+ + File.separator
+ + "1.bin.tmp");
+ if (objectTmpFile.exists() && objectTmpFile.isFile()) {
+ success = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ Assert.assertFalse(success);
+ }
+
+ protected String convertPathString(String path) {
+ return BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8));
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 8de74e066ab..e3ebd991cf0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2959,7 +2959,9 @@ public class DataRegion implements IDataRegionForQuery {
matchedObjectDir.toPath(),
Integer.MAX_VALUE,
(path, attrs) ->
- attrs.isRegularFile() && path.getFileName().toString().endsWith(".bin"))) {
+ attrs.isRegularFile()
+ && (path.getFileName().toString().endsWith(".bin")
+ || path.getFileName().toString().endsWith(".tmp")))) {
paths.forEach(
path -> {
Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path);
@@ -2985,7 +2987,7 @@ public class DataRegion implements IDataRegionForQuery {
relativePath.getName(relativePath.getNameCount() - 2).toString()),
StandardCharsets.UTF_8);
String fileName = path.getFileName().toString();
- long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.')));
+ long timestamp = Long.parseLong(fileName.substring(0, fileName.indexOf('.')));
logger.info(
"timestamp {}, measurementId {}, ideviceId {}",
timestamp,
(iotdb) 04/05: optimize drop table
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_delet
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6bbf453f62f3804aca11fac272b9413a55d1ca4d
Author: HTHou <ha...@apache.org>
AuthorDate: Thu Dec 18 12:19:32 2025 +0800
optimize drop table
---
.../db/storageengine/dataregion/DataRegion.java | 109 +++++++++++----------
.../dataregion/modification/DeletionPredicate.java | 4 +
.../modification/TableDeletionEntry.java | 9 ++
.../db/storageengine/rescon/disk/TierManager.java | 11 +++
4 files changed, 82 insertions(+), 51 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 27a290b60fd..8de74e066ab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2766,10 +2766,21 @@ public class DataRegion implements IDataRegionForQuery {
}
}
- List<File> matchedObjectDirs =
+ List<File> objectTableDirs =
TierManager.getInstance().getAllMatchedObjectDirs(dataRegionIdString, tableName);
- if (!matchedObjectDirs.isEmpty()) {
- deleteObjectFiles(matchedObjectDirs, modEntries);
+ if (!objectTableDirs.isEmpty()) {
+ boolean droppingTable = false;
+ for (TableDeletionEntry entry : modEntries) {
+ if (entry.isDroppingTable()) {
+ for (File objectTableDir : objectTableDirs) {
+ droppingTable = true;
+ FileUtils.deleteQuietly(objectTableDir);
+ }
+ }
+ }
+ if (!droppingTable) {
+ deleteObjectFiles(objectTableDirs, modEntries);
+ }
}
List<List<TsFileResource>> sealedTsFileResourceLists = new ArrayList<>(modEntries.size());
@@ -2943,54 +2954,50 @@ public class DataRegion implements IDataRegionForQuery {
private void deleteObjectFiles(List<File> matchedObjectDirs, List<TableDeletionEntry> modEntries)
throws IOException {
for (File matchedObjectDir : matchedObjectDirs) {
- try (Stream<Path> paths = Files.walk(matchedObjectDir.toPath())) {
- paths
- .filter(Files::isRegularFile)
- .filter(
- path -> {
- String name = path.getFileName().toString();
- return name.endsWith(".bin");
- })
- .forEach(
- path -> {
- Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path);
- String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2];
- for (int i = 0; i < ideviceIdSegments.length; i++) {
- ideviceIdSegments[i] =
- config.getRestrictObjectLimit()
- ? relativePath.getName(i).toString()
- : new String(
- BaseEncoding.base32()
- .omitPadding()
- .decode(relativePath.getName(i).toString()),
- StandardCharsets.UTF_8);
- }
- IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments);
- String measurementId =
- config.getRestrictObjectLimit()
- ? relativePath.getName(relativePath.getNameCount() - 2).toString()
- : new String(
- BaseEncoding.base32()
- .omitPadding()
- .decode(
- relativePath
- .getName(relativePath.getNameCount() - 2)
- .toString()),
- StandardCharsets.UTF_8);
- String fileName = path.getFileName().toString();
- long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.')));
- logger.info(
- "timestamp {}, measurementId {}, ideviceId {}",
- timestamp,
- measurementId,
- iDeviceID);
- for (TableDeletionEntry modEntry : modEntries) {
- if (modEntry.affects(iDeviceID, timestamp, timestamp)
- && modEntry.affects(measurementId)) {
- ObjectTypeUtils.deleteObjectPath(path.toFile());
- }
- }
- });
+ try (Stream<Path> paths =
+ Files.find(
+ matchedObjectDir.toPath(),
+ Integer.MAX_VALUE,
+ (path, attrs) ->
+ attrs.isRegularFile() && path.getFileName().toString().endsWith(".bin"))) {
+ paths.forEach(
+ path -> {
+ Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path);
+ String[] ideviceIdSegments = new String[relativePath.getNameCount() - 2];
+ for (int i = 0; i < ideviceIdSegments.length; i++) {
+ ideviceIdSegments[i] =
+ config.getRestrictObjectLimit()
+ ? relativePath.getName(i).toString()
+ : new String(
+ BaseEncoding.base32()
+ .omitPadding()
+ .decode(relativePath.getName(i).toString()),
+ StandardCharsets.UTF_8);
+ }
+ IDeviceID iDeviceID = Factory.DEFAULT_FACTORY.create(ideviceIdSegments);
+ String measurementId =
+ config.getRestrictObjectLimit()
+ ? relativePath.getName(relativePath.getNameCount() - 2).toString()
+ : new String(
+ BaseEncoding.base32()
+ .omitPadding()
+ .decode(
+ relativePath.getName(relativePath.getNameCount() - 2).toString()),
+ StandardCharsets.UTF_8);
+ String fileName = path.getFileName().toString();
+ long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.')));
+ logger.info(
+ "timestamp {}, measurementId {}, ideviceId {}",
+ timestamp,
+ measurementId,
+ iDeviceID);
+ for (TableDeletionEntry modEntry : modEntries) {
+ if (modEntry.affects(iDeviceID, timestamp, timestamp)
+ && modEntry.affects(measurementId)) {
+ ObjectTypeUtils.deleteObjectPath(path.toFile());
+ }
+ }
+ });
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java
index 7e79e8f580d..0a22da2a90a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java
@@ -72,6 +72,10 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable
this.idPredicate = idPredicate;
}
+ public IDPredicate getIdPredicate() {
+ return idPredicate;
+ }
+
public String getTableName() {
return tableName;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java
index 858b6645b2f..61ec2e0d678 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.modification;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper;
+import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.IDPredicateType;
import org.apache.iotdb.db.utils.ModificationUtils;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -136,6 +137,14 @@ public class TableDeletionEntry extends ModEntry {
return predicate.getTableName();
}
+ public boolean isDroppingTable() {
+ IDPredicate idPredicate = predicate.getIdPredicate();
+ return idPredicate.type == IDPredicateType.NOP
+ && predicate.getMeasurementNames().isEmpty()
+ && timeRange.getMin() == Long.MIN_VALUE
+ && timeRange.getMax() == Long.MAX_VALUE;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index 3aff31a9631..f83bdc36b4b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -287,6 +287,17 @@ public class TierManager {
public List<File> getAllMatchedObjectDirs(String regionIdStr, String... path) {
List<File> matchedDirs = new ArrayList<>();
+ boolean hasObjectDir = false;
+ for (String objectDir : objectDirs) {
+ File objectDirPath = FSFactoryProducer.getFSFactory().getFile(objectDir);
+ if (objectDirPath.exists()) {
+ hasObjectDir = true;
+ break;
+ }
+ }
+ if (!hasObjectDir) {
+ return matchedDirs;
+ }
StringBuilder objectPath = new StringBuilder();
objectPath.append(regionIdStr);
for (String str : path) {
(iotdb) 03/05: remove empty object dir
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_delet
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fdbbd52a12cea171dffe424784da81c9cd8b7f24
Author: HTHou <ha...@apache.org>
AuthorDate: Tue Dec 16 16:40:28 2025 +0800
remove empty object dir
---
.../iotdb/db/storageengine/rescon/disk/TierManager.java | 11 +++++++++++
.../java/org/apache/iotdb/db/utils/ObjectTypeUtils.java | 13 ++++++++++++-
2 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index a673e44dff9..3aff31a9631 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -39,6 +39,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileStore;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
@@ -174,6 +175,16 @@ public class TierManager {
} catch (DiskSpaceInsufficientException e) {
logger.error("All disks of tier {} are full.", tierLevel, e);
}
+ // try to remove empty objectDirs
+ for (String dir : objectDirs) {
+ File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
+ if (dirFile.isDirectory() && Objects.requireNonNull(dirFile.list()).length == 0) {
+ try {
+ Files.delete(dirFile.toPath());
+ } catch (IOException ignore) {
+ }
+ }
+ }
}
tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
index 2a03facbf96..15964ffaddd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
@@ -55,6 +55,7 @@ import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.Optional;
public class ObjectTypeUtils {
@@ -310,8 +311,18 @@ public class ObjectTypeUtils {
logger.error("Failed to remove object file {}", file.getAbsolutePath(), e);
}
}
- if (file.getParentFile().exists()) {
+ deleteEmptyParentDir(file);
+ }
+ private static void deleteEmptyParentDir(File file) {
+ File dir = file.getParentFile();
+ if (dir.isDirectory() && Objects.requireNonNull(dir.list()).length == 0) {
+ try {
+ Files.deleteIfExists(dir.toPath());
+ deleteEmptyParentDir(dir);
+ } catch (IOException e) {
+ logger.error("Failed to remove empty object dir {}", dir.getAbsolutePath(), e);
+ }
}
}