You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2022/03/17 07:46:59 UTC
[iotdb] 01/45: use wildcard to query
This is an automated email from the ASF dual-hosted git repository.
jianyun pushed a commit to branch rocksdb/dev
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5d015a1aca975d239293b3b1003bf0d96fd07b97
Author: lisijia <li...@360.cn>
AuthorDate: Wed Mar 2 10:49:33 2022 +0800
use wildcard to query
---
.../iotdb/db/metadata/rocksdb/MRocksDBManager.java | 84 +++++++++++++++++++---
.../iotdb/db/metadata/rocksdb/RocksDBUtils.java | 41 +----------
.../db/metadata/rocksdb/MRocksDBUnitTest.java | 2 +-
3 files changed, 80 insertions(+), 47 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBManager.java
index 743e34a..37821ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBManager.java
@@ -109,12 +109,27 @@ import java.util.Stack;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
+import java.util.function.Consumer;
import java.util.function.Function;
import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.*;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.DATA_BLOCK_TYPE_SCHEMA;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.DEFAULT_ALIGNED_ENTITY_VALUE;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.DEFAULT_NODE_VALUE;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.FLAG_IS_ALIGNED;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.FLAG_IS_SCHEMA;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.FLAG_SET_TTL;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.NODE_TYPE_ENTITY;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.NODE_TYPE_MEASUREMENT;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.NODE_TYPE_SG;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.TABLE_NAME_TAGS;
+import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.ZERO;
+import static org.apache.iotdb.db.metadata.rocksdb.RocksDBUtils.getAllCompoundMode;
+import static org.apache.iotdb.db.metadata.rocksdb.RocksDBUtils.newStringArray;
+import static org.apache.iotdb.db.metadata.rocksdb.RocksDBUtils.replaceWildcard;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
/**
@@ -940,19 +955,57 @@ public class MRocksDBManager implements IMetaManager {
@Override
public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
throws MetadataException {
+ return getCountByNodeType(new Character[] {NODE_TYPE_MEASUREMENT}, pathPattern.getNodes());
+ }
- return getKeyNumByPrefix(pathPattern, NODE_TYPE_MEASUREMENT, isPrefixMatch).size();
+ // eg. root.a.*.**.b.**.c
+ public void replaceMultiWildcard(
+ String[] nodes, int maxLevel, Consumer<String> consumer, Character[] nodeTypeArray)
+ throws IllegalPathException {
+ List<Integer> multiWildcardPosition = new ArrayList<>();
+ for (int i = 0; i < nodes.length; i++) {
+ if (MULTI_LEVEL_PATH_WILDCARD.equals(nodes[i])) {
+ multiWildcardPosition.add(i);
+ }
+ }
+ if (multiWildcardPosition.isEmpty()) {
+ traverseByPatternPath(nodes, consumer, nodeTypeArray);
+ } else if (multiWildcardPosition.size() == 1) {
+ for (int i = 1; i <= maxLevel - nodes.length + 2; i++) {
+ String[] clone = nodes.clone();
+ clone[multiWildcardPosition.get(0)] = replaceWildcard(i);
+ traverseByPatternPath(newStringArray(clone), consumer, nodeTypeArray);
+ }
+ } else {
+ for (int sum = multiWildcardPosition.size();
+ sum <= maxLevel - (nodes.length - multiWildcardPosition.size() - 1);
+ sum++) {
+ List<int[]> result = getAllCompoundMode(sum, multiWildcardPosition.size());
+ for (int[] value : result) {
+ String[] clone = nodes.clone();
+ for (int i = 0; i < value.length; i++) {
+ clone[multiWildcardPosition.get(i)] = replaceWildcard(value[i]);
+ }
+ traverseByPatternPath(newStringArray(clone), consumer, nodeTypeArray);
+ }
+ }
+ }
}
- public void traverseByPatternPath(PartialPath pathPattern) {
- String[] nodes = pathPattern.getNodes();
+ public void traverseByPatternPath(
+ String[] nodes, Consumer<String> consumer, Character[] nodeTypeArray) {
+ // String[] nodes = pathPattern.getNodes();
int startIndex = 0;
List<String[]> scanKeys = new ArrayList<>();
int indexOfPrefix = indexOfFirstWildcard(nodes, startIndex);
if (indexOfPrefix >= nodes.length) {
- System.out.println("matched key: " + pathPattern.getFullPath());
+ StringBuilder stringBuilder = new StringBuilder();
+ for (int i = 0; i < nodes.length; i++) {
+ stringBuilder.append(RockDBConstants.PATH_SEPARATOR).append(nodes[i]);
+ }
+ consumer.accept(stringBuilder.substring(1));
return;
}
@@ -979,7 +1032,7 @@ public class MRocksDBManager implements IMetaManager {
prefixNodes -> {
String levelPrefix =
RocksDBUtils.getLevelPathPrefix(prefixNodes, prefixNodes.length - 1, level);
- Arrays.stream(ALL_NODE_TYPE_ARRAY)
+ Arrays.stream(nodeTypeArray)
.parallel()
.forEach(
x -> {
@@ -993,6 +1046,7 @@ public class MRocksDBManager implements IMetaManager {
if (RocksDBUtils.suffixMatch(iterator.key(), suffixToMatch)) {
if (lastIteration) {
System.out.println("matched key: " + new String(iterator.key()));
+ consumer.accept(new String(iterator.key()));
} else {
tempNodes.add(RocksDBUtils.toMetaNodes(iterator.key()));
}
@@ -1120,7 +1174,21 @@ public class MRocksDBManager implements IMetaManager {
@Override
public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch)
throws MetadataException {
- return getKeyNumByPrefix(pathPattern, NODE_TYPE_ENTITY, isPrefixMatch).size();
+ return getCountByNodeType(new Character[] {NODE_TYPE_ENTITY}, pathPattern.getNodes());
+ }
+
+ private int getCountByNodeType(Character[] nodetype, String[] nodes) throws IllegalPathException {
+ AtomicInteger atomicInteger = new AtomicInteger(0);
+ Consumer<String> consumer =
+ new Consumer<String>() {
+ @Override
+ public void accept(String s) {
+ atomicInteger.incrementAndGet();
+ }
+ };
+
+ replaceMultiWildcard(nodes, MAX_PATH_DEPTH, consumer, nodetype);
+ return atomicInteger.get();
}
@Override
@@ -1136,7 +1204,7 @@ public class MRocksDBManager implements IMetaManager {
@Override
public int getStorageGroupNum(PartialPath pathPattern, boolean isPrefixMatch)
throws MetadataException {
- return getKeyNumByPrefix(pathPattern, NODE_TYPE_SG, isPrefixMatch).size();
+ return getCountByNodeType(new Character[] {NODE_TYPE_SG}, pathPattern.getNodes());
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RocksDBUtils.java b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RocksDBUtils.java
index d3b585c..7cf4b76 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RocksDBUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rocksdb/RocksDBUtils.java
@@ -43,7 +43,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.iotdb.db.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.db.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_ROOT;
import static org.apache.iotdb.db.metadata.rocksdb.RockDBConstants.DATA_BLOCK_TYPE_ALIAS;
@@ -472,41 +471,7 @@ public class RocksDBUtils {
return stringBuilder.toString();
}
- // eg. root.a.*.**.b.**.c
- public static List<String[]> replaceMultiWildcard(String[] nodes, int maxLevel)
- throws IllegalPathException {
- List<String[]> allResult = new ArrayList<>();
- List<Integer> multiWildcardPosition = new ArrayList<>();
- for (int i = 0; i < nodes.length; i++) {
- if (MULTI_LEVEL_PATH_WILDCARD.equals(nodes[i])) {
- multiWildcardPosition.add(i);
- }
- }
- if (multiWildcardPosition.isEmpty()) {
- allResult.add(nodes);
- } else if (multiWildcardPosition.size() == 1) {
- for (int i = 1; i <= maxLevel - nodes.length + 2; i++) {
- String[] clone = nodes.clone();
- clone[multiWildcardPosition.get(0)] = replaceWildcard(i);
- allResult.add(newStringArray(clone));
- }
- } else {
- List<int[]> result =
- getAllCompoundMode(
- maxLevel - (nodes.length - multiWildcardPosition.size() - 1),
- multiWildcardPosition.size());
- for (int[] value : result) {
- String[] clone = nodes.clone();
- for (int i = 0; i < value.length; i++) {
- clone[multiWildcardPosition.get(i)] = replaceWildcard(value[i]);
- }
- allResult.add(newStringArray(clone));
- }
- }
- return allResult;
- }
-
- private static String[] newStringArray(String[] oldArray) throws IllegalPathException {
+ public static String[] newStringArray(String[] oldArray) throws IllegalPathException {
StringBuilder stringBuilder = new StringBuilder();
for (String str : oldArray) {
stringBuilder.append(PATH_SEPARATOR).append(str);
@@ -514,7 +479,7 @@ public class RocksDBUtils {
return MetaUtils.splitPathToDetachedPath(stringBuilder.substring(1));
}
- private static String replaceWildcard(int num) {
+ public static String replaceWildcard(int num) {
StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < num; i++) {
stringBuilder.append(RockDBConstants.PATH_SEPARATOR).append(ONE_LEVEL_PATH_WILDCARD);
@@ -522,7 +487,7 @@ public class RocksDBUtils {
return stringBuilder.substring(1);
}
- private static List<int[]> getAllCompoundMode(int sum, int n) {
+ public static List<int[]> getAllCompoundMode(int sum, int n) {
if (n <= 2) {
List<int[]> result = new ArrayList<>();
for (int i = 1; i < sum; i++) {
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java
index df6cfd2..a0ce63f 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/rocksdb/MRocksDBUnitTest.java
@@ -216,7 +216,7 @@ public class MRocksDBUnitTest {
path, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED, null, null);
}
- mRocksDBManager.traverseByPatternPath(new PartialPath("root.sg.d1.*"));
+ // mRocksDBManager.traverseByPatternPath(new PartialPath("root.sg.d1.*"));
}
@After