You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2022/03/24 10:49:07 UTC
[iotdb] 01/02: [rocksdb] remove getInstance and use uuid to create directory. URL: https://geelib.qihoo.net/geelib/repoMergeDetail?repoId=tsdb/iotdb&mergeId=9
This is an automated email from the ASF dual-hosted git repository.
jianyun pushed a commit to branch feature/rocksdb
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c7cc8c52d4f452affbf94bc6966c349cce8dee89
Author: lisijia <li...@360.cn>
AuthorDate: Thu Mar 24 18:33:57 2022 +0800
[rocksdb] remove getInstance and use uuid to create directory.
URL: https://geelib.qihoo.net/geelib/repoMergeDetail?repoId=tsdb/iotdb&mergeId=9
---
.../iotdb/db/metadata/rocksdb/RSchemaEngine.java | 51 ++++++++++++++++------
.../metadata/rocksdb/RSchemaReadWriteHandler.java | 27 ++++++------
.../db/metadata/rocksdb/mnode/REntityMNode.java | 9 ++--
.../db/metadata/rocksdb/mnode/RInternalMNode.java | 5 ++-
.../iotdb/db/metadata/rocksdb/mnode/RMNode.java | 16 +++----
.../metadata/rocksdb/mnode/RMeasurementMNode.java | 10 +++--
.../metadata/rocksdb/mnode/RStorageGroupMNode.java | 11 +++--
.../db/metadata/rocksdb/MRocksDBUnitTest.java | 9 ++--
8 files changed, 83 insertions(+), 55 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaEngine.java
index d8252d6..e247ad2 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaEngine.java
@@ -163,7 +163,17 @@ public class RSchemaEngine implements ISchemaEngine {
public RSchemaEngine() throws MetadataException {
try {
- readWriteHandler = RSchemaReadWriteHandler.getInstance();
+ readWriteHandler = new RSchemaReadWriteHandler();
+ } catch (RocksDBException e) {
+ logger.error("create RocksDBReadWriteHandler fail", e);
+ throw new MetadataException(e);
+ }
+ }
+
+ @TestOnly
+ public RSchemaEngine(String path) throws MetadataException {
+ try {
+ readWriteHandler = new RSchemaReadWriteHandler(path);
} catch (RocksDBException e) {
logger.error("create RocksDBReadWriteHandler fail", e);
throw new MetadataException(e);
@@ -615,7 +625,7 @@ public class RSchemaEngine implements ISchemaEngine {
Lock lock = locksPool.computeIfAbsent(levelPath, x -> new ReentrantLock());
if (lock.tryLock(MAX_LOCK_WAIT_TIME, TimeUnit.MILLISECONDS)) {
try {
- deletedNode = new RMeasurementMNode(path, value);
+ deletedNode = new RMeasurementMNode(path, value, readWriteHandler);
WriteBatch batch = new WriteBatch();
// delete the last node of path
batch.delete(key);
@@ -1493,7 +1503,8 @@ public class RSchemaEngine implements ISchemaEngine {
BiFunction<byte[], byte[], Boolean> function =
(a, b) -> {
allResult.add(
- new RMeasurementMNode(RSchemaUtils.getPathByInnerName(new String(a)), b)
+ new RMeasurementMNode(
+ RSchemaUtils.getPathByInnerName(new String(a)), b, readWriteHandler)
.getMeasurementPath());
return true;
};
@@ -1509,7 +1520,8 @@ public class RSchemaEngine implements ISchemaEngine {
BiFunction<byte[], byte[], Boolean> function =
(a, b) -> {
MeasurementPath measurementPath =
- new RMeasurementMNode(RSchemaUtils.getPathByInnerName(new String(a)), b)
+ new RMeasurementMNode(
+ RSchemaUtils.getPathByInnerName(new String(a)), b, readWriteHandler)
.getMeasurementPath();
Object tag = RSchemaUtils.parseNodeValue(b, RMNodeValueType.TAGS);
if (!(tag instanceof Map)) {
@@ -1649,7 +1661,9 @@ public class RSchemaEngine implements ISchemaEngine {
if (readWriteHandler.keyExistByType(levelPath, RMNodeType.STORAGE_GROUP, holder)) {
node =
new RStorageGroupMNode(
- MetaUtils.getStorageGroupPathByLevel(path, i).getFullPath(), holder.getValue());
+ MetaUtils.getStorageGroupPathByLevel(path, i).getFullPath(),
+ holder.getValue(),
+ readWriteHandler);
break;
}
}
@@ -1676,7 +1690,9 @@ public class RSchemaEngine implements ISchemaEngine {
}
result.add(
new RStorageGroupMNode(
- RSchemaUtils.getPathByInnerName(new String(iterator.key())), iterator.value()));
+ RSchemaUtils.getPathByInnerName(new String(iterator.key())),
+ iterator.value(),
+ readWriteHandler));
}
return result;
}
@@ -1688,7 +1704,7 @@ public class RSchemaEngine implements ISchemaEngine {
Holder<byte[]> holder = new Holder<>();
try {
if (readWriteHandler.keyExistByType(levelPath, RMNodeType.ENTITY, holder)) {
- return new REntityMNode(path.getFullPath(), holder.getValue());
+ return new REntityMNode(path.getFullPath(), holder.getValue(), readWriteHandler);
} else {
throw new PathNotExistException(path.getFullPath());
}
@@ -1727,14 +1743,16 @@ public class RSchemaEngine implements ISchemaEngine {
try {
Holder<byte[]> holder = new Holder<>();
if (readWriteHandler.keyExistByType(key, RMNodeType.MEASUREMENT, holder)) {
- node = new RMeasurementMNode(fullPath.getFullPath(), holder.getValue());
+ node = new RMeasurementMNode(fullPath.getFullPath(), holder.getValue(), readWriteHandler);
} else if (readWriteHandler.keyExistByType(key, RMNodeType.ALISA, holder)) {
byte[] aliasValue = holder.getValue();
if (aliasValue != null) {
ByteBuffer byteBuffer = ByteBuffer.wrap(aliasValue);
ReadWriteIOUtils.readBytes(byteBuffer, 3);
byte[] oriKey = RSchemaUtils.readOriginKey(byteBuffer);
- node = new RMeasurementMNode(fullPath.getFullPath(), readWriteHandler.get(null, oriKey));
+ node =
+ new RMeasurementMNode(
+ fullPath.getFullPath(), readWriteHandler.get(null, oriKey), readWriteHandler);
}
}
return node;
@@ -1876,7 +1894,8 @@ public class RSchemaEngine implements ISchemaEngine {
}
byte[] originValue = holder.getValue();
- RMeasurementMNode mNode = new RMeasurementMNode(path.getFullPath(), originValue);
+ RMeasurementMNode mNode =
+ new RMeasurementMNode(path.getFullPath(), originValue, readWriteHandler);
if (mNode.getAttributes() != null) {
for (Map.Entry<String, String> entry : attributesMap.entrySet()) {
if (mNode.getAttributes().containsKey(entry.getKey())) {
@@ -1917,7 +1936,8 @@ public class RSchemaEngine implements ISchemaEngine {
throw new PathNotExistException(path.getFullPath());
}
byte[] originValue = holder.getValue();
- RMeasurementMNode mNode = new RMeasurementMNode(path.getFullPath(), originValue);
+ RMeasurementMNode mNode =
+ new RMeasurementMNode(path.getFullPath(), originValue, readWriteHandler);
boolean hasTags = false;
if (mNode.getTags() != null && mNode.getTags().size() > 0) {
hasTags = true;
@@ -1966,7 +1986,8 @@ public class RSchemaEngine implements ISchemaEngine {
}
byte[] originValue = holder.getValue();
- RMeasurementMNode mNode = new RMeasurementMNode(path.getFullPath(), originValue);
+ RMeasurementMNode mNode =
+ new RMeasurementMNode(path.getFullPath(), originValue, readWriteHandler);
int tagLen = mNode.getTags() == null ? 0 : mNode.getTags().size();
boolean didAnyUpdate = false;
@@ -2018,7 +2039,8 @@ public class RSchemaEngine implements ISchemaEngine {
if (lock.tryLock(MAX_LOCK_WAIT_TIME, TimeUnit.MILLISECONDS)) {
try {
byte[] originValue = holder.getValue();
- RMeasurementMNode mNode = new RMeasurementMNode(path.getFullPath(), originValue);
+ RMeasurementMNode mNode =
+ new RMeasurementMNode(path.getFullPath(), originValue, readWriteHandler);
boolean didAnyUpdate = false;
for (Map.Entry<String, String> entry : alterMap.entrySet()) {
didAnyUpdate = false;
@@ -2072,7 +2094,8 @@ public class RSchemaEngine implements ISchemaEngine {
throw new PathNotExistException(path.getFullPath());
}
byte[] originValue = holder.getValue();
- RMeasurementMNode mNode = new RMeasurementMNode(path.getFullPath(), originValue);
+ RMeasurementMNode mNode =
+ new RMeasurementMNode(path.getFullPath(), originValue, readWriteHandler);
boolean didAnyUpdate = false;
if (mNode.getTags() != null && mNode.getTags().containsKey(oldKey)) {
String value = mNode.getTags().get(oldKey);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaReadWriteHandler.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaReadWriteHandler.java
index e8bdd8f..94f9dc2 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaReadWriteHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RSchemaReadWriteHandler.java
@@ -97,8 +97,6 @@ public class RSchemaReadWriteHandler {
private RocksDB rocksDB;
- private static volatile RSchemaReadWriteHandler readWriteHandler;
-
ConcurrentMap<String, ColumnFamilyHandle> columnFamilyHandleMap = new ConcurrentHashMap<>();
List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
@@ -107,7 +105,16 @@ public class RSchemaReadWriteHandler {
RocksDB.loadLibrary();
}
+ @TestOnly
+ public RSchemaReadWriteHandler(String path) throws RocksDBException {
+ initReadWriteHandler(path);
+ }
+
public RSchemaReadWriteHandler() throws RocksDBException {
+ initReadWriteHandler(ROCKSDB_PATH);
+ }
+
+ private void initReadWriteHandler(String path) throws RocksDBException {
Options options = new Options();
org.rocksdb.Logger rocksDBLogger = new RSchemaLogger(options, logger);
rocksDBLogger.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
@@ -137,17 +144,17 @@ public class RSchemaReadWriteHandler {
DBOptions dbOptions = new DBOptions(options);
- initColumnFamilyDescriptors(options);
+ initColumnFamilyDescriptors(options, path);
- rocksDB = RocksDB.open(dbOptions, ROCKSDB_PATH, columnFamilyDescriptors, columnFamilyHandles);
+ rocksDB = RocksDB.open(dbOptions, path, columnFamilyDescriptors, columnFamilyHandles);
initInnerColumnFamilies();
initRootKey();
}
- private void initColumnFamilyDescriptors(Options options) throws RocksDBException {
- List<byte[]> cfs = RocksDB.listColumnFamilies(options, ROCKSDB_PATH);
+ private void initColumnFamilyDescriptors(Options options, String path) throws RocksDBException {
+ List<byte[]> cfs = RocksDB.listColumnFamilies(options, path);
if (cfs.size() <= 0) {
cfs = new ArrayList<>();
cfs.add(RocksDB.DEFAULT_COLUMN_FAMILY);
@@ -581,13 +588,5 @@ public class RSchemaReadWriteHandler {
public void close() throws RocksDBException {
rocksDB.syncWal();
rocksDB.closeE();
- readWriteHandler = null;
- }
-
- public static RSchemaReadWriteHandler getInstance() throws RocksDBException {
- if (readWriteHandler == null) {
- readWriteHandler = new RSchemaReadWriteHandler();
- }
- return readWriteHandler;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/REntityMNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/REntityMNode.java
index 6514b89..34dcb59 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/REntityMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/REntityMNode.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.metadata.logfile.MLogWriter;
import org.apache.iotdb.db.metadata.mnode.IEntityMNode;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.rocksdb.RSchemaConstants;
+import org.apache.iotdb.db.metadata.rocksdb.RSchemaReadWriteHandler;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
@@ -38,12 +39,12 @@ public class REntityMNode extends RInternalMNode implements IEntityMNode {
*
* @param fullPath
*/
- public REntityMNode(String fullPath) {
- super(fullPath);
+ public REntityMNode(String fullPath, RSchemaReadWriteHandler readWriteHandler) {
+ super(fullPath, readWriteHandler);
}
- public REntityMNode(String fullPath, byte[] value) {
- super(fullPath);
+ public REntityMNode(String fullPath, byte[] value, RSchemaReadWriteHandler readWriteHandler) {
+ super(fullPath, readWriteHandler);
deserialize(value);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RInternalMNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RInternalMNode.java
index aa3ff08..7fdc67e 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RInternalMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RInternalMNode.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.db.metadata.logfile.MLogWriter;
import org.apache.iotdb.db.metadata.mnode.IMNode;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.rocksdb.RSchemaConstants;
+import org.apache.iotdb.db.metadata.rocksdb.RSchemaReadWriteHandler;
import org.apache.iotdb.db.metadata.rocksdb.RSchemaUtils;
import org.apache.iotdb.db.metadata.template.Template;
@@ -42,8 +43,8 @@ public class RInternalMNode extends RMNode {
*
* @param fullPath
*/
- public RInternalMNode(String fullPath) {
- super(fullPath);
+ public RInternalMNode(String fullPath, RSchemaReadWriteHandler readWriteHandler) {
+ super(fullPath, readWriteHandler);
}
/** check whether the MNode has a child with the name */
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMNode.java
index d631b7c..6228d39 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMNode.java
@@ -49,14 +49,10 @@ public abstract class RMNode implements IMNode {
protected static final Logger logger = LoggerFactory.getLogger(RMNode.class);
/** Constructor of MNode. */
- public RMNode(String fullPath) {
+ public RMNode(String fullPath, RSchemaReadWriteHandler readWriteHandler) {
this.fullPath = fullPath.intern();
this.name = fullPath.substring(fullPath.lastIndexOf(RSchemaConstants.PATH_SEPARATOR) + 1);
- try {
- readWriteHandler = RSchemaReadWriteHandler.getInstance();
- } catch (RocksDBException e) {
- logger.error("create RocksDBReadWriteHandler fail", e);
- }
+ this.readWriteHandler = readWriteHandler;
}
@Override
@@ -95,16 +91,16 @@ public abstract class RMNode implements IMNode {
if (value != null) {
switch (type.getValue()) {
case RSchemaConstants.NODE_TYPE_SG:
- node = new RStorageGroupMNode(keyName, value);
+ node = new RStorageGroupMNode(keyName, value, readWriteHandler);
return node;
case RSchemaConstants.NODE_TYPE_INTERNAL:
- node = new RInternalMNode(keyName);
+ node = new RInternalMNode(keyName, readWriteHandler);
return node;
case RSchemaConstants.NODE_TYPE_ENTITY:
- node = new REntityMNode(keyName, value);
+ node = new REntityMNode(keyName, value, readWriteHandler);
return node;
case RSchemaConstants.NODE_TYPE_MEASUREMENT:
- node = new RMeasurementMNode(keyName, value);
+ node = new RMeasurementMNode(keyName, value, readWriteHandler);
return node;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMeasurementMNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMeasurementMNode.java
index 8657d5b..48b83e1 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMeasurementMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RMeasurementMNode.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.metadata.mnode.IMNode;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.path.MeasurementPath;
import org.apache.iotdb.db.metadata.rocksdb.RSchemaConstants;
+import org.apache.iotdb.db.metadata.rocksdb.RSchemaReadWriteHandler;
import org.apache.iotdb.db.metadata.rocksdb.RSchemaUtils;
import org.apache.iotdb.db.metadata.template.Template;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -52,12 +53,13 @@ public class RMeasurementMNode extends RMNode implements IMeasurementMNode {
*
* @param fullPath
*/
- public RMeasurementMNode(String fullPath) {
- super(fullPath);
+ public RMeasurementMNode(String fullPath, RSchemaReadWriteHandler readWriteHandler) {
+ super(fullPath, readWriteHandler);
}
- public RMeasurementMNode(String fullPath, byte[] value) {
- super(fullPath);
+ public RMeasurementMNode(
+ String fullPath, byte[] value, RSchemaReadWriteHandler readWriteHandler) {
+ super(fullPath, readWriteHandler);
deserialize(value);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RStorageGroupMNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RStorageGroupMNode.java
index 850e39f..61a0a86 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RStorageGroupMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/mnode/RStorageGroupMNode.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.metadata.SchemaRegion;
import org.apache.iotdb.db.metadata.logfile.MLogWriter;
import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
+import org.apache.iotdb.db.metadata.rocksdb.RSchemaReadWriteHandler;
import org.apache.iotdb.db.metadata.rocksdb.RSchemaUtils;
import java.io.IOException;
@@ -35,13 +36,15 @@ public class RStorageGroupMNode extends RInternalMNode implements IStorageGroupM
*
* @param fullPath
*/
- public RStorageGroupMNode(String fullPath, long dataTTL) {
- super(fullPath);
+ public RStorageGroupMNode(
+ String fullPath, long dataTTL, RSchemaReadWriteHandler readWriteHandler) {
+ super(fullPath, readWriteHandler);
this.dataTTL = dataTTL;
}
- public RStorageGroupMNode(String fullPath, byte[] value) {
- super(fullPath);
+ public RStorageGroupMNode(
+ String fullPath, byte[] value, RSchemaReadWriteHandler readWriteHandler) {
+ super(fullPath, readWriteHandler);
Object ttl = RSchemaUtils.parseNodeValue(value, RMNodeValueType.TTL);
if (ttl == null) {
ttl = IoTDBDescriptor.getInstance().getConfig().getDefaultTTL();
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java
index ae3ea47..179d947 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java
@@ -36,20 +36,23 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
import static org.apache.iotdb.db.metadata.rocksdb.RSchemaReadWriteHandler.ROCKSDB_PATH;
public class MRocksDBUnitTest {
private RSchemaEngine RSchemaEngine;
+ private String rocksdbPath;
@Before
public void setUp() throws MetadataException {
- File file = new File(ROCKSDB_PATH);
+ rocksdbPath = ROCKSDB_PATH + UUID.randomUUID();
+ File file = new File(rocksdbPath);
if (!file.exists()) {
file.mkdirs();
}
- RSchemaEngine = new RSchemaEngine();
+ RSchemaEngine = new RSchemaEngine(rocksdbPath);
}
@Test
@@ -329,7 +332,7 @@ public class MRocksDBUnitTest {
}
public void resetEnv() {
- File rockdDbFile = new File(ROCKSDB_PATH);
+ File rockdDbFile = new File(rocksdbPath);
if (rockdDbFile.exists() && rockdDbFile.isDirectory()) {
FileUtils.deleteDirectory(rockdDbFile);
}