You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by he...@apache.org on 2023/05/29 03:38:21 UTC
[iotdb] branch tiered_storage updated: add dn_default_space_move_thresholds config
This is an automated email from the ASF dual-hosted git repository.
heiming pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tiered_storage by this push:
new 88e3960090b add dn_default_space_move_thresholds config
88e3960090b is described below
commit 88e3960090b50ec2902e0cc64ed4cf03872dde02
Author: HeimingZ <zh...@qq.com>
AuthorDate: Mon May 29 11:38:04 2023 +0800
add dn_default_space_move_thresholds config
---
.../resources/conf/iotdb-common.properties | 13 +++--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 59 ++++++++++++++++------
.../db/engine/migration/MigrationTaskManager.java | 9 ++--
.../engine/migration/LocalMigrationTaskTest.java | 27 ++++++++++
.../engine/migration/RemoteMigrationTaskTest.java | 27 ++++++++++
6 files changed, 120 insertions(+), 25 deletions(-)
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index b6b5e246742..49f9b993c7e 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -176,6 +176,11 @@ cluster_name=defaultCluster
# Datatype: int
# migrate_thread_count=3
+# Disk move threshold at which current tier data will be moved to the next tier
+# If tiered storage is enabled, please separate TTL of different tiers by semicolons ";".
+# Datatype: double(percentage)
+# dn_default_space_move_thresholds=0.15
+
# Disk remaining threshold at which DataNode is set to ReadOnly status
# Datatype: double(percentage)
# disk_space_warning_threshold=0.05
@@ -1182,11 +1187,11 @@ cluster_name=defaultCluster
# Datatype: string
# remote_tsfile_cache_dirs=data/datanode/data/cache
-# Datatype: long
-# remote_tsfile_cache_max_disk_usage=53687091200
-
# Datatype: int
-# remote_tsfile_cache_page_size=20971520
+# remote_tsfile_cache_page_size_in_kb=20971520
+
+# Datatype: long
+# remote_tsfile_cache_max_disk_usage_in_mb=53687091200
# Datatype: string
# object_storage_name=AWS_S3
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index e52f0b1af04..e1008ae92a1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1140,6 +1140,8 @@ public class IoTDBConfig {
/** Threads for migration tasks */
private int migrateThreadCount = 1;
+ private double[] spaceMoveThresholds = {0.15};
+
/** Enable hdfs or not */
private boolean enableObjectStorage = true;
@@ -3948,6 +3950,14 @@ public class IoTDBConfig {
this.migrateThreadCount = migrateThreadCount;
}
+ public double[] getSpaceMoveThresholds() {
+ return spaceMoveThresholds;
+ }
+
+ public void setSpaceMoveThresholds(double[] spaceMoveThresholds) {
+ this.spaceMoveThresholds = spaceMoveThresholds;
+ }
+
public boolean isEnableObjectStorage() {
return enableObjectStorage;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 6c5987cf955..f998f8b3eec 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1082,6 +1082,9 @@ public class IoTDBDescriptor {
// author cache
loadAuthorCache(properties);
+ // object storage
+ loadMigrationProps(properties);
+
// object storage
loadObjectStorageProps(properties);
@@ -1153,35 +1156,61 @@ public class IoTDBDescriptor {
"author_cache_expire_time", String.valueOf(conf.getAuthorCacheExpireTime()))));
}
+ private void loadMigrationProps(Properties properties) {
+ conf.setMigrateThreadCount(
+ Integer.parseInt(
+ properties.getProperty(
+ "migrate_thread_count", String.valueOf(conf.getMigrateThreadCount()))));
+
+ String[] moveThresholdsStr = new String[conf.getSpaceMoveThresholds().length];
+ for (int i = 0; i < moveThresholdsStr.length; ++i) {
+ moveThresholdsStr[i] = String.valueOf(conf.getSpaceMoveThresholds()[i]);
+ }
+ moveThresholdsStr =
+ properties
+ .getProperty(
+ "dn_default_space_move_thresholds",
+ String.join(IoTDBConstant.TIER_SEPARATOR, moveThresholdsStr))
+ .split(IoTDBConstant.TIER_SEPARATOR);
+ double[] moveThresholds = new double[moveThresholdsStr.length];
+ for (int i = 0; i < moveThresholds.length; ++i) {
+ moveThresholds[i] = Double.parseDouble(moveThresholdsStr[i]);
+ if (moveThresholds[i] < 0) {
+ moveThresholds[i] = 0.15;
+ }
+ }
+ conf.setSpaceMoveThresholds(moveThresholds);
+ }
+
private void loadObjectStorageProps(Properties properties) {
conf.setEnableObjectStorage(
Boolean.parseBoolean(
properties.getProperty(
"enable_object_storage", String.valueOf(conf.isEnableObjectStorage()))));
- conf.setCacheDirs(
- properties
- .getProperty("remote_tsfile_cache_dirs", String.join(",", conf.getCacheDirs()))
- .split(","));
- conf.setCacheMaxDiskUsage(
- Long.parseLong(
- properties.getProperty(
- "remote_tsfile_cache_max_disk_usage",
- String.valueOf(conf.getCacheMaxDiskUsage()))));
- conf.setCachePageSize(
- Integer.parseInt(
- properties.getProperty(
- "remote_tsfile_cache_page_size", String.valueOf(conf.getCachePageSize()))));
conf.setObjectStorageName(
properties.getProperty("object_storage_name", conf.getObjectStorageName()));
- conf.setObjectStorageEndpoint(
- properties.getProperty("object_storage_endpoint", conf.getObjectStorageEndpoint()));
conf.setObjectStorageBucket(
properties.getProperty("object_storage_bucket", conf.getObjectStorageBucket()));
+ conf.setObjectStorageEndpoint(
+ properties.getProperty("object_storage_endpoint", conf.getObjectStorageEndpoint()));
conf.setObjectStorageAccessKey(
properties.getProperty("object_storage_access_key", conf.getObjectStorageAccessKey()));
conf.setObjectStorageAccessSecret(
properties.getProperty(
"object_storage_access_secret", conf.getObjectStorageAccessSecret()));
+ conf.setCacheDirs(
+ properties
+ .getProperty("remote_tsfile_cache_dirs", String.join(",", conf.getCacheDirs()))
+ .split(","));
+ conf.setCachePageSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "remote_tsfile_cache_page_size_in_kb", String.valueOf(conf.getCachePageSize()))));
+ conf.setCacheMaxDiskUsage(
+ Long.parseLong(
+ properties.getProperty(
+ "remote_tsfile_cache_max_disk_usage_in_mb",
+ String.valueOf(conf.getCacheMaxDiskUsage()))));
}
private void loadWALProps(Properties properties) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTaskManager.java b/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTaskManager.java
index a67a5358f04..2f9dbee2a64 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTaskManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/migration/MigrationTaskManager.java
@@ -53,10 +53,7 @@ public class MigrationTaskManager implements IService {
private static final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig();
private static final TierManager tierManager = TierManager.getInstance();
private static final long CHECK_INTERVAL_IN_SECONDS = 10;
- private static final double TIER_DISK_SPACE_WARN_THRESHOLD =
- commonConfig.getDiskSpaceWarningThreshold() + 0.1;
- private static final double TIER_DISK_SPACE_SAFE_THRESHOLD =
- commonConfig.getDiskSpaceWarningThreshold() + 0.2;
+ private static final double MOVE_THRESHOLD_SAFE_LIMIT = 0.1;
private static final int MIGRATION_TASK_LIMIT = 20;
/** max concurrent migration tasks */
private final AtomicInteger migrationTasksNum = new AtomicInteger(0);
@@ -93,7 +90,7 @@ public class MigrationTaskManager implements IService {
public MigrationScheduleTask() {
for (int i = 0; i < tierManager.getTiersNum(); i++) {
double usage = tierDiskUsableSpace[i] * 1.0 / tierDiskTotalSpace[i];
- if (usage <= TIER_DISK_SPACE_WARN_THRESHOLD) {
+ if (usage <= iotdbConfig.getSpaceMoveThresholds()[i]) {
needMigrationTiers.add(i);
}
}
@@ -158,7 +155,7 @@ public class MigrationTaskManager implements IService {
tierDiskUsableSpace[tierLevel] -= sourceTsFile.getTsFileSize();
if (needMigrationTiers.contains(tierLevel)) {
double usage = tierDiskUsableSpace[tierLevel] * 1.0 / tierDiskTotalSpace[tierLevel];
- if (usage > TIER_DISK_SPACE_SAFE_THRESHOLD) {
+ if (usage > iotdbConfig.getSpaceMoveThresholds()[tierLevel] + MOVE_THRESHOLD_SAFE_LIMIT) {
needMigrationTiers.remove(tierLevel);
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/migration/LocalMigrationTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/migration/LocalMigrationTaskTest.java
new file mode 100644
index 00000000000..fac7ec3eb61
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/engine/migration/LocalMigrationTaskTest.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.migration;
+
+import org.junit.Test;
+
+public class LocalMigrationTaskTest {
+
+ @Test
+ public void migrate() {}
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTaskTest.java
new file mode 100644
index 00000000000..0569924ae26
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/engine/migration/RemoteMigrationTaskTest.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.migration;
+
+import org.junit.Test;
+
+public class RemoteMigrationTaskTest {
+
+ @Test
+ public void migrate() {}
+}