You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xu...@apache.org on 2020/12/25 02:19:17 UTC
[iotdb] branch virtual_partition_2_merge updated: add config and
SuppressWarnings
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a commit to branch virtual_partition_2_merge
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/virtual_partition_2_merge by this push:
new de9e7ef add config and SuppressWarnings
de9e7ef is described below
commit de9e7ef350494bba5c6790caee8e3e75fe791871
Author: 151250176 <15...@smail.nju.edu.cn>
AuthorDate: Fri Dec 25 10:18:49 2020 +0800
add config and SuppressWarnings
---
server/src/assembly/resources/conf/iotdb-engine.properties | 5 +++++
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 ++++++------
.../main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 8 ++++++++
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 ++++++--
.../main/java/org/apache/iotdb/db/engine/StorageEngine.java | 9 ++++++++-
.../storagegroup/virtualSg/HashVirtualPartitioner.java | 2 +-
.../storagegroup/virtualSg/VirtualStorageGroupManager.java | 2 ++
.../org/apache/iotdb/db/integration/IoTDBCompleteIT.java | 2 +-
.../iotdb/db/integration/IoTDBLoadExternalTsfileIT.java | 6 +++---
.../apache/iotdb/db/sync/receiver/load/FileLoaderTest.java | 3 +--
10 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 538d972..b6de9c6 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -230,6 +230,11 @@ mtree_snapshot_interval=100000
# Only take effect when enable_mtree_snapshot=true.
mtree_snapshot_threshold_time=3600
+# number of virtual storage group
+# virtual storage group is the unit of concurrency in memory
+# recommend value is [storage group number] * [virtual storage group number] = virtual cpu core number
+virtual_storage_group_num = 8
+
####################
### Memory Control Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index b5103ef..dafe59a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -817,9 +817,9 @@ public class IoTDBConfig {
private boolean debugState = false;
/**
- * the number of virtual partition
+ * the number of virtual storage group
*/
- private int virtualPartitionNum = 8;
+ private int virtualStorageGroupNum = 8;
public IoTDBConfig() {
// empty constructor
@@ -2174,12 +2174,12 @@ public class IoTDBConfig {
this.defaultIndexWindowRange = defaultIndexWindowRange;
}
- public int getVirtualPartitionNum() {
- return virtualPartitionNum;
+ public int getVirtualStorageGroupNum() {
+ return virtualStorageGroupNum;
}
- public void setVirtualPartitionNum(int virtualPartitionNum) {
- this.virtualPartitionNum = virtualPartitionNum;
+ public void setVirtualStorageGroupNum(int virtualStorageGroupNum) {
+ this.virtualStorageGroupNum = virtualStorageGroupNum;
}
public boolean isRpcAdvancedCompressionEnable() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index 9d7efd1..f19db87 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -81,6 +81,9 @@ public class IoTDBConfigCheck {
private static String maxDegreeOfIndexNode = String
.valueOf(TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode());
+ private static final String VIRTUAL_STORAGE_GROUP_NUM = "virtual_storage_group_num";
+ private static String virtualStorageGroupNum = String.valueOf(config.getVirtualStorageGroupNum());
+
private static final String IOTDB_VERSION_STRING = "iotdb_version";
public static IoTDBConfigCheck getInstance() {
@@ -131,6 +134,7 @@ public class IoTDBConfigCheck {
systemProperties.put(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
systemProperties.put(TAG_ATTRIBUTE_SIZE_STRING, tagAttributeTotalSize);
systemProperties.put(MAX_DEGREE_OF_INDEX_STRING, maxDegreeOfIndexNode);
+ systemProperties.put(VIRTUAL_STORAGE_GROUP_NUM, virtualStorageGroupNum);
}
@@ -298,6 +302,10 @@ public class IoTDBConfigCheck {
if (!(properties.getProperty(MAX_DEGREE_OF_INDEX_STRING).equals(maxDegreeOfIndexNode))) {
printErrorLogAndExit(MAX_DEGREE_OF_INDEX_STRING);
}
+
+ if (!(properties.getProperty(VIRTUAL_STORAGE_GROUP_NUM).equals(virtualStorageGroupNum))) {
+ printErrorLogAndExit(VIRTUAL_STORAGE_GROUP_NUM);
+ }
}
private void printErrorLogAndExit(String property) {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 00d5b1d..52f31c3 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -45,8 +45,8 @@ import org.slf4j.LoggerFactory;
public class IoTDBDescriptor {
private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
- private IoTDBConfig conf = new IoTDBConfig();
private static CommandLine commandLine;
+ private IoTDBConfig conf = new IoTDBConfig();
protected IoTDBDescriptor() {
loadProps();
@@ -240,7 +240,7 @@ public class IoTDBDescriptor {
}
int mlogBufferSize = Integer.parseInt(properties.getProperty("mlog_buffer_size",
- Integer.toString(conf.getMlogBufferSize())));
+ Integer.toString(conf.getMlogBufferSize())));
if (mlogBufferSize > 0) {
conf.setMlogBufferSize(mlogBufferSize);
}
@@ -438,6 +438,7 @@ public class IoTDBDescriptor {
.parseInt(properties.getProperty("performance_stat_memory_in_kb",
Integer.toString(conf.getPerformanceStatMemoryInKB())).trim()));
+
int maxConcurrentClientNum = Integer.parseInt(properties.
getProperty("rpc_max_concurrent_client_num",
Integer.toString(conf.getRpcMaxConcurrentClientNum()).trim()));
@@ -530,6 +531,9 @@ public class IoTDBDescriptor {
conf.setDebugState(Boolean.parseBoolean(properties
.getProperty("debug_state", String.valueOf(conf.isDebugOn()))));
+ conf.setVirtualStorageGroupNum(Integer.parseInt(properties
+ .getProperty("virtual_storage_group_num",
+ String.valueOf(conf.getVirtualStorageGroupNum()))));
// mqtt
if (properties.getProperty(IoTDBConstant.MQTT_HOST_NAME) != null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 864f75e..44999fc 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -394,7 +394,14 @@ public class StorageEngine implements IService {
}
}
- // get storage group processor by partial path
+ /**
+ * get storage group processor by path
+ * @param storageGroupPath path of the storage group
+ * @param storageGroupMNode mnode of the storage group
+ * @return found or new storage group processor
+ */
+ @SuppressWarnings("java:S2445")
+ // actually storageGroupMNode is a unique object on the mtree, synchronize it is reasonable
private StorageGroupProcessor getStorageGroupProcessorByPath(PartialPath storageGroupPath,
StorageGroupMNode storageGroupMNode)
throws StorageGroupProcessorException, StorageEngineException {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java
index 0fa6ff0..33d05be 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/HashVirtualPartitioner.java
@@ -25,7 +25,7 @@ import org.apache.iotdb.db.utils.TestOnly;
public class HashVirtualPartitioner implements VirtualPartitioner {
public static int STORAGE_GROUP_NUM = IoTDBDescriptor.getInstance().getConfig()
- .getVirtualPartitionNum();
+ .getVirtualStorageGroupNum();
private HashVirtualPartitioner() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java
index 7112fa8..446dbc7 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/virtualSg/VirtualStorageGroupManager.java
@@ -93,6 +93,8 @@ public class VirtualStorageGroupManager {
* @param partialPath device path
* @return virtual storage group processor
*/
+ @SuppressWarnings("java:S2445")
+ // actually storageGroupMNode is a unique object on the mtree, synchronize it is reasonable
public StorageGroupProcessor getProcessor(PartialPath partialPath, StorageGroupMNode storageGroupMNode)
throws StorageGroupProcessorException, StorageEngineException {
int loc = partitioner.deviceToStorageGroup(partialPath);
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
index 2b18511..bbcecf9 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java
@@ -47,7 +47,7 @@ public class IoTDBCompleteIT {
@After
public void tearDown() throws Exception {
EnvironmentUtils.cleanEnv();
- HashVirtualPartitioner.getInstance().setStorageGroupNum(IoTDBDescriptor.getInstance().getConfig().getVirtualPartitionNum());
+ HashVirtualPartitioner.getInstance().setStorageGroupNum(IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum());
}
@Test
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
index bc12a39..a0e0206 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
@@ -129,11 +129,11 @@ public class IoTDBLoadExternalTsfileIT {
public void setUp() throws Exception {
IoTDBDescriptor.getInstance().getConfig()
.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
- IoTDBDescriptor.getInstance().getConfig().setVirtualPartitionNum(1);
+ IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(1);
HashVirtualPartitioner.getInstance().setStorageGroupNum(1);
EnvironmentUtils.closeStatMonitor();
EnvironmentUtils.envSetUp();
- virtualPartitionNum = IoTDBDescriptor.getInstance().getConfig().getVirtualPartitionNum();
+ virtualPartitionNum = IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum();
Class.forName(Config.JDBC_DRIVER_NAME);
prepareData(insertSequenceSqls);
}
@@ -143,7 +143,7 @@ public class IoTDBLoadExternalTsfileIT {
EnvironmentUtils.cleanEnv();
IoTDBDescriptor.getInstance().getConfig()
.setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION);
- IoTDBDescriptor.getInstance().getConfig().setVirtualPartitionNum(virtualPartitionNum);
+ IoTDBDescriptor.getInstance().getConfig().setVirtualStorageGroupNum(virtualPartitionNum);
HashVirtualPartitioner.getInstance().setStorageGroupNum(virtualPartitionNum);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java b/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java
index 6b3d53e..b05d441 100644
--- a/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/sync/receiver/load/FileLoaderTest.java
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -88,7 +87,7 @@ public class FileLoaderTest {
EnvironmentUtils.cleanEnv();
IoTDBDescriptor.getInstance().getConfig().setSyncEnable(false);
HashVirtualPartitioner.getInstance()
- .setStorageGroupNum(IoTDBDescriptor.getInstance().getConfig().getVirtualPartitionNum());
+ .setStorageGroupNum(IoTDBDescriptor.getInstance().getConfig().getVirtualStorageGroupNum());
}
@Test