You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2025/05/17 14:52:41 UTC
(iotdb) branch rc/1.3.4-1 created (now 7d54943c0a1)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch rc/1.3.4-1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 7d54943c0a1 update RN
This branch includes the following new commits:
new 4e62af2c26c [To dev/1.3] Use reference time position for PartitionTableAutoCleaner
new d44d2bbc0df [To dev/1.3] Fix auto cleaning partition table bug
new 7d54943c0a1 update RN
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 02/03: [To dev/1.3] Fix auto cleaning partition table bug
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.4-1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d44d2bbc0dfdfb9860fd060f78621541018bb918
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Wed May 14 19:41:33 2025 +0800
[To dev/1.3] Fix auto cleaning partition table bug
---
.../iotdb/commons/partition/SeriesPartitionTable.java | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java
index f99d06c9c0d..f46344566dc 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java
@@ -22,14 +22,15 @@ package org.apache.iotdb.commons.partition;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils;
+import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TProtocol;
import org.apache.tsfile.utils.ReadWriteIOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
@@ -49,7 +50,12 @@ import java.util.stream.Collectors;
public class SeriesPartitionTable {
- private static final Logger LOGGER = LoggerFactory.getLogger(SeriesPartitionTable.class);
+ // should only be used in CN scope, in DN scope should directly use
+ // TimePartitionUtils.getTimePartitionInterval()
+ private static final long TIME_PARTITION_INTERVAL =
+ CommonDateTimeUtils.convertMilliTimeWithPrecision(
+ TimePartitionUtils.getTimePartitionInterval(),
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
private final ConcurrentSkipListMap<TTimePartitionSlot, List<TConsensusGroupId>>
seriesPartitionMap;
@@ -255,7 +261,8 @@ public class SeriesPartitionTable {
while (iterator.hasNext()) {
Map.Entry<TTimePartitionSlot, List<TConsensusGroupId>> entry = iterator.next();
TTimePartitionSlot timePartitionSlot = entry.getKey();
- if (timePartitionSlot.getStartTime() + TTL < currentTimeSlot.getStartTime()) {
+ if (timePartitionSlot.getStartTime() + TIME_PARTITION_INTERVAL + TTL
+ <= currentTimeSlot.getStartTime()) {
removedTimePartitions.add(timePartitionSlot);
iterator.remove();
}
(iotdb) 01/03: [To dev/1.3] Use reference time position for PartitionTableAutoCleaner
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.4-1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4e62af2c26c2eb6918848e597c14478eb6df9126
Author: Yongzao <yo...@apache.org>
AuthorDate: Tue May 13 17:07:53 2025 +0800
[To dev/1.3] Use reference time position for PartitionTableAutoCleaner
---
.../partition/IoTDBPartitionTableAutoCleanIT.java | 7 +++--
....java => IoTDBPartitionTableAutoCleanUSIT.java} | 30 +++++++++++++---------
.../iotdb/confignode/manager/TTLManager.java | 9 +++++--
.../procedure/PartitionTableAutoCleaner.java | 29 ++++++++++++++++++---
.../iotdb/commons/utils/TimePartitionUtils.java | 13 ----------
5 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
index c62739cdc13..a7addb87c1d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -52,7 +51,11 @@ public class IoTDBPartitionTableAutoCleanIT {
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ / TEST_TIME_PARTITION_INTERVAL
+ * TEST_TIME_PARTITION_INTERVAL);
private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
@Before
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
similarity index 84%
copy from integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
copy to integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
index c62739cdc13..f8e94c28dcb 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionTableAutoCleanUSIT.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.it.partition;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.it.env.EnvFactory;
@@ -43,17 +42,22 @@ import java.util.concurrent.TimeUnit;
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
-public class IoTDBPartitionTableAutoCleanIT {
+public class IoTDBPartitionTableAutoCleanUSIT {
private static final String TREE_DATABASE_PREFIX = "root.db.g_";
private static final int TEST_REPLICATION_FACTOR = 1;
- private static final long TEST_TIME_PARTITION_INTERVAL = 604800000;
+ private static final long TEST_TIME_PARTITION_INTERVAL_IN_MS = 604800_000;
private static final long TEST_TTL_CHECK_INTERVAL = 5_000;
private static final TTimePartitionSlot TEST_CURRENT_TIME_SLOT =
- TimePartitionUtils.getCurrentTimePartitionSlot();
- private static final long TEST_TTL = 7 * TEST_TIME_PARTITION_INTERVAL;
+ new TTimePartitionSlot()
+ .setStartTime(
+ System.currentTimeMillis()
+ * 1000L
+ / TEST_TIME_PARTITION_INTERVAL_IN_MS
+ * TEST_TIME_PARTITION_INTERVAL_IN_MS);
+ private static final long TEST_TTL_IN_MS = 7 * TEST_TIME_PARTITION_INTERVAL_IN_MS;
@Before
public void setUp() throws Exception {
@@ -62,8 +66,10 @@ public class IoTDBPartitionTableAutoCleanIT {
.getCommonConfig()
.setSchemaReplicationFactor(TEST_REPLICATION_FACTOR)
.setDataReplicationFactor(TEST_REPLICATION_FACTOR)
- .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL)
- .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL);
+ .setTimePartitionInterval(TEST_TIME_PARTITION_INTERVAL_IN_MS)
+ .setTTLCheckInterval(TEST_TTL_CHECK_INTERVAL)
+ // Note that the time precision of IoTDB is us in this IT
+ .setTimestampPrecision("us");
// Init 1C1D environment
EnvFactory.getEnv().initClusterEnvironment(1, 1);
@@ -89,7 +95,7 @@ public class IoTDBPartitionTableAutoCleanIT {
statement.execute(
String.format(
"INSERT INTO %s(timestamp, s) VALUES (%d, %d)",
- databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL * 2, -1));
+ databaseName, TEST_CURRENT_TIME_SLOT.getStartTime() - TEST_TTL_IN_MS * 2000, -1));
// Insert existed data
statement.execute(
String.format(
@@ -97,15 +103,15 @@ public class IoTDBPartitionTableAutoCleanIT {
databaseName, TEST_CURRENT_TIME_SLOT.getStartTime(), 1));
}
// Let db0.TTL > device.TTL, the valid TTL should be the bigger one
- statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s0 %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
statement.execute(String.format("SET TTL TO %s0.s %d", TREE_DATABASE_PREFIX, 10));
// Let db1.TTL < device.TTL, the valid TTL should be the bigger one
statement.execute(String.format("SET TTL TO %s1 %d", TREE_DATABASE_PREFIX, 10));
- statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(String.format("SET TTL TO %s1.s %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
// Set TTL to path db2.**
- statement.execute(String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL));
+ statement.execute(
+ String.format("SET TTL TO %s2.** %d", TREE_DATABASE_PREFIX, TEST_TTL_IN_MS));
}
-
TDataPartitionReq req = new TDataPartitionReq();
for (int i = 0; i < 3; i++) {
req.putToPartitionSlotsMap(String.format("%s%d", TREE_DATABASE_PREFIX, i), new TreeMap<>());
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
index 55a61389126..b0c4dd5f296 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
@@ -135,8 +136,12 @@ public class TTLManager {
* @return the maximum ttl of the subtree of the corresponding database. return NULL_TTL if the
* TTL is not set or the database does not exist.
*/
- public long getDatabaseMaxTTL(String database) {
- return ttlInfo.getDatabaseMaxTTL(database);
+ public long getDatabaseMaxTTL(final String database) {
+ final long ttl = ttlInfo.getDatabaseMaxTTL(database);
+ return ttl == Long.MAX_VALUE || ttl < 0
+ ? ttl
+ : CommonDateTimeUtils.convertMilliTimeWithPrecision(
+ ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
}
/** Only used for upgrading from old database-level ttl to device-level ttl. */
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
index ce5d07276db..8a08e5efe99 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.confignode.procedure;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.consensus.exception.ConsensusException;
@@ -42,6 +41,10 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
private static final Logger LOGGER = LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
+
+ private static final String timestampPrecision =
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+
private final ConfigManager configManager;
public PartitionTableAutoCleaner(ConfigManager configManager) {
@@ -59,6 +62,12 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
for (String database : databases) {
long databaseTTL = configManager.getTTLManager().getDatabaseMaxTTL(database);
databaseTTLMap.put(database, databaseTTL);
+ }
+ LOGGER.info(
+ "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner, databaseTTL: {}",
+ databaseTTLMap);
+ for (String database : databases) {
+ long databaseTTL = databaseTTLMap.get(database);
if (!configManager.getPartitionManager().isDatabaseExist(database)
|| databaseTTL < 0
|| databaseTTL == Long.MAX_VALUE) {
@@ -71,8 +80,7 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
"[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner for: {}",
databaseTTLMap);
// Only clean the partition table when necessary
- TTimePartitionSlot currentTimePartitionSlot =
- TimePartitionUtils.getCurrentTimePartitionSlot();
+ TTimePartitionSlot currentTimePartitionSlot = getCurrentTimePartitionSlot();
try {
configManager
.getConsensusManager()
@@ -82,4 +90,19 @@ public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
}
}
}
+
+ /**
+ * @return The time partition slot corresponding to current timestamp. Note that we do not shift
+ * the start time to the correct starting point, since this interface only constructs a time
+ * reference position for the partition table cleaner.
+ */
+ private static TTimePartitionSlot getCurrentTimePartitionSlot() {
+ if ("ms".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis());
+ } else if ("us".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
+ } else {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
+ }
+ }
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
index ed01e8a5f3f..7b331fddaac 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/TimePartitionUtils.java
@@ -34,9 +34,6 @@ public class TimePartitionUtils {
private static long timePartitionOrigin =
CommonDescriptor.getInstance().getConfig().getTimePartitionOrigin();
- private static String timestampPrecision =
- CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
-
/** Time range for dividing database, the time unit is the same with IoTDB's TimestampPrecision */
private static long timePartitionInterval =
CommonDescriptor.getInstance().getConfig().getTimePartitionInterval();
@@ -74,16 +71,6 @@ public class TimePartitionUtils {
}
}
- public static TTimePartitionSlot getCurrentTimePartitionSlot() {
- if ("ms".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis());
- } else if ("us".equals(timestampPrecision)) {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000);
- } else {
- return getTimePartitionSlot(System.currentTimeMillis() * 1000_000);
- }
- }
-
public static TTimePartitionSlot getTimePartitionSlot(long time) {
TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot();
timePartitionSlot.setStartTime(getTimePartitionLowerBound(time));
(iotdb) 03/03: update RN
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.4-1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7d54943c0a11203d4a041e6479e9d9f1a2b434b0
Author: HTHou <hh...@outlook.com>
AuthorDate: Sat May 17 22:52:24 2025 +0800
update RN
---
RELEASE_NOTES.md | 5 +++++
iotdb-core/ainode/pyproject.toml | 4 ++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index a397608990f..ed6bae86d29 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -18,6 +18,11 @@
under the License.
-->
+# Apache IoTDB 1.3.4-1
+
+This is a bug-fix version of 1.3.4
+
+- Fix the bug that will remove the data partition table by mistake in case of us/ns time precision and using ttl
# Apache IoTDB 1.3.4
diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml
index dd3252d424a..f3bf4251773 100644
--- a/iotdb-core/ainode/pyproject.toml
+++ b/iotdb-core/ainode/pyproject.toml
@@ -21,7 +21,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "apache-iotdb-ainode"
-version = "1.3.4"
+version = "1.3.4-1"
description = "Apache IoTDB AINode"
readme = "README.md"
authors = ["Apache Software Foundation <de...@iotdb.apache.org>"]
@@ -63,4 +63,4 @@ pmdarima = "^2.0.4"
hmmlearn = "^0.3.0"
[tool.poetry.scripts]
-ainode = "iotdb.ainode.script:main"
\ No newline at end of file
+ainode = "iotdb.ainode.script:main"