You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2025/05/13 06:49:57 UTC
(iotdb) branch partition-table-cleaner-US-IT created (now dfed424b81c)
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a change to branch partition-table-cleaner-US-IT
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at dfed424b81c Finish
This branch includes the following new commits:
new dfed424b81c Finish
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Finish
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch partition-table-cleaner-US-IT
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dfed424b81c95b787343978090571abdea9fefa5
Author: YongzaoDan <53...@qq.com>
AuthorDate: Tue May 13 14:49:44 2025 +0800
Finish
---
.../partition/IoTDBPartitionTableAutoCleanIT.java | 7 +++--
....java => IoTDBPartitionTableAutoCleanUSIT.java} | 34 +++++++++++++---------
.../procedure/PartitionTableAutoCleaner.java | 23 +++++++++++++--
.../iotdb/commons/utils/TimePartitionUtils.java | 13 ---------
4 files changed, 45 insertions(+), 32 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
index eb6cdbddb2a..3f3596746f0 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -54,7 +53,11 @@ public class IoTDBPartitionTableAutoCleanIT {
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ / TEST_TIME_PARTITION_INTERVAL
+ * TEST_TIME_PARTITION_INTERVAL);
private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
@Before
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
similarity index 87%
copy from integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
copy to integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
index eb6cdbddb2a..69ab3a03d28 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -44,18 +43,23 @@ import java.util.concurrent.TimeUnit;
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
-public class IoTDBPartitionTableAutoCleanIT {
+public class IoTDBPartitionTableAutoCleanUSIT {
private static final String TREE_DATABASE_PREFIX = "root.db.g_";
private static final String TABLE_DATABASE_PREFIX = "database_";
private static final int TEST_REPLICATION_FACTOR = 1;
- private static final long TEST_TIME_PARTITION_INTERVAL = 604800000;
+ private static final long TEST_TIME_PARTITION_INTERVAL_IN_MS = 604800_000;
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
- private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ * 1000L
+ / TEST_TIME_PARTITION_INTERVAL_IN_MS
+ * TEST_TIME_PARTITION_INTERVAL_IN_MS);
+ private static final long TEST_TTL_IN_MS = 7 * TEST_TIME_PARTITION_INTERVAL_IN_MS;
@Before
public void setUp() throws Exception {
@@ -64,8 +68,10 @@ public class IoTDBPartitionTableAutoCleanIT {
.getCommonConfig()
.setSchemaReplicationFactor(TEST_REPLICATION_FACTOR)
.setDataReplicationFactor(TEST_REPLICATION_FACTOR)
- .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL)
- .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL);
+ .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL_IN_MS)
+ .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL)
+ // Note that the time precision of IoTDB is us in this IT
+ .setTimestampPrecision("us");
// Init 1C1D environment
EnvFactory.getEnv().initClusterEnvironment(1, 1);
@@ -91,7 +97,7 @@ public class IoTDBPartitionTableAutoCleanIT {
statement.execute(
String.format(
"INSERT INTO %s(timestamp, s) VALUES (%d, %d)",
- databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1));
+ databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL_IN_MS * 2000, -1));
// Insert existed data
statement.execute(
String.format(
@@ -99,15 +105,15 @@ public class IoTDBPartitionTableAutoCleanIT {
databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1));
}
// Let db0.TTL > device.TTL, the valid TTL should be the bigger one
- statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
statement.execute(String.format("SET TTL TO %s0.s %d", TREE_DATABASE_PREFIX, 10));
// Let db1.TTL < device.TTL, the valid TTL should be the bigger one
statement.execute(String.format("SET TTL TO %s1 %d", TREE_DATABASE_PREFIX, 10));
- statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
// Set TTL to path db2.**
- statement.execute(String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(
+ String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
}
-
TDataPartitionReq req = new TDataPartitionReq();
for (int i = 0; i < 3; i++) {
req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>());
@@ -146,13 +152,13 @@ public class IoTDBPartitionTableAutoCleanIT {
statement.execute(
String.format(
"INSERT INTO tb(time, s) VALUES (%d, %d)",
- TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1));
+ TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL_IN_MS * 2000, -1));
// Insert existed data
statement.execute(
String.format(
"INSERT INTO tb(time, s) VALUES (%d, %d)", TEST_CURRENT_TIME_SLOT.getStartTime(), 1));
statement.execute(String.format("USE %s", TABLE_DATABASE_PREFIX));
- statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d", TEST_TTL));
+ statement.execute(String.format("ALTER TABLE tb SET PROPERTIES TTL=%d", TEST_TTL_IN_MS));
}
TDataPartitionReq req = new TDataPartitionReq();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
index a0baa3a77c7..2ab15283207 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.utils.PathUtils;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.consensus.exception.ConsensusException;
@@ -43,6 +42,10 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
private static final Logger LOGGER = LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
+
+ private static final String timestampPrecision =
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+
private final ConfigManager configManager;
public PartitionTableAutoCleaner(ConfigManager configManager) {
@@ -81,8 +84,7 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
"[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner for: {}",
databaseTTLMap);
// Only clean the partition table when necessary
- TTimePartitionSlot currentTimePartitionSlot =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ TTimePartitionSlot currentTimePartitionSlot = getCurrentTimePartitionSlot();
try {
configManager
.getConsensusManager()
@@ -92,4 +94,19 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
}
}
}
+
+ /**
+ * @return The time partition slot corresponding to current timestamp. Note that we do not shift
+ * the start time to the correct starting point, since this interface only constructs a time
+ * reference position for the partition table cleaner.
+ */
+ private static TTimePartitionSlot getCurrentTimePartitionSlot() {
+ if ("ms".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis());
+ } else if ("us".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
+ } else {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
+ }
+ }
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
index 058f7433929..eb53cdb2798 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
@@ -34,9 +34,6 @@ public class TimePartitionUtils {
private static long timePartitionOrigin =
CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin();
- private static String timestampPrecision =
- CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
-
/** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */
private static long timePartitionInterval =
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval();
@@ -74,16 +71,6 @@ public class TimePartitionUtils {
}
}
- public static TTimePartitionSlot getCurrentTimePartitionSlot() {
- if ("ms".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis());
- } else if ("us".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000);
- } else {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000_000);
- }
- }
-
public static TTimePartitionSlot getTimePartitionSlot(long time) {
TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot();
timePartitionSlot.setStartTime(getTimePartitionLowerBound(time));