You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/10/18 02:53:35 UTC
[iotdb] branch clusterVector created (now 1f7aff3)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch clusterVector
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 1f7aff3 implement raw vector query without value filter
This branch includes the following new commits:
new 334209f Fix cluster vector meta error
new 69f2d96 Merge branch 'master' into clusterVector
new 1f7aff3 implement raw vector query without value filter
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 03/03: implement raw vector query without value filter
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch clusterVector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1f7aff37e9c4ecbf24fd9fe5d1c8a2e4d8031c8a
Author: Alima777 <wx...@gmail.com>
AuthorDate: Mon Oct 18 10:15:01 2021 +0800
implement raw vector query without value filter
---
.../java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java | 7 +++++--
.../apache/iotdb/cluster/query/reader/ClusterReaderFactory.java | 7 ++++---
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java
index 36a56e8..32677d4 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java
@@ -306,13 +306,16 @@ public class LocalQueryExecutor {
.forEach(
fullPath -> {
try {
- if (fullPath.contains("$#$")) {
+ System.out.println("After RPC: " + fullPath);
+ if (fullPath.startsWith("VP$")) {
String[] array = fullPath.split(":");
+ String vectorId = array[0].replace("VP$", "");
+ System.out.println("After RPC: " + vectorId);
List<String> subSensorsList = new ArrayList<>();
for (int i = 1; i < array.length; i++) {
subSensorsList.add(array[i]);
}
- paths.add(new VectorPartialPath(array[0], subSensorsList));
+ paths.add(new VectorPartialPath(vectorId, subSensorsList));
} else {
paths.add(new PartialPath(fullPath));
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
index d692830..d55c510 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
@@ -708,12 +708,12 @@ public class ClusterReaderFactory {
paths.forEach(
path -> {
if (path instanceof VectorPartialPath) {
- StringBuilder builder = new StringBuilder(path.getFullPath());
+ System.out.println("Before RPC" + path);
+ StringBuilder builder = new StringBuilder("VP$");
+ builder.append(path.getFullPath());
List<String> subSensorsList = ((VectorPartialPath) path).getSubSensorsList();
for (String subSensor : subSensorsList) {
builder.append(":");
- builder.append(path.getFullPath());
- builder.append(".");
builder.append(subSensor);
}
fullPaths.add(builder.toString());
@@ -752,6 +752,7 @@ public class ClusterReaderFactory {
if (valueFilter != null) {
request.setValueFilterBytes(SerializeUtils.serializeFilter(valueFilter));
}
+ System.out.println("Before RPC: " + path);
request.setPath(path.getFullPath());
request.setHeader(partitionGroup.getHeader());
request.setQueryId(context.getQueryId());
[iotdb] 01/03: Fix cluster vector meta error
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch clusterVector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 334209f5c4954023e14d1c644266619a18cc36af
Author: Alima777 <wx...@gmail.com>
AuthorDate: Tue Oct 12 10:53:47 2021 +0800
Fix cluster vector meta error
---
.../apache/iotdb/cluster/metadata/CMManager.java | 41 +++++++++++++---------
.../cluster/client/sync/SyncClientAdaptorTest.java | 4 ++-
.../iotdb/AlignedTimeseriesSessionExample.java | 25 +++++--------
thrift-cluster/src/main/thrift/cluster.thrift | 1 +
4 files changed, 36 insertions(+), 35 deletions(-)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
index f28308d..c5034a2 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/metadata/CMManager.java
@@ -196,6 +196,7 @@ public class CMManager extends MManager {
}
String measurement = fullPath.getMeasurement();
+ System.out.println("FullPath: " + fullPath.getExactFullPath());
if (fullPath instanceof VectorPartialPath) {
if (((VectorPartialPath) fullPath).getSubSensorsList().size() != 1) {
return TSDataType.VECTOR;
@@ -203,6 +204,7 @@ public class CMManager extends MManager {
measurement = ((VectorPartialPath) fullPath).getSubSensor(0);
}
}
+ System.out.println("Measurement: " + measurement);
// try remote cache first
try {
@@ -229,18 +231,17 @@ public class CMManager extends MManager {
MeasurementMNode.getMeasurementMNode(
null, measurementSchema.getMeasurementId(), measurementSchema, null);
if (measurementSchema instanceof VectorMeasurementSchema) {
- for (int i = 0; i < measurementSchema.getSubMeasurementsList().size(); i++) {
+ for (String subMeasurement : measurementSchema.getSubMeasurementsList()) {
cacheMeta(
- ((VectorPartialPath) fullPath).getPathWithSubSensor(i), measurementMNode, false);
+ new VectorPartialPath(fullPath.getDevice(), subMeasurement),
+ measurementMNode,
+ false);
}
- cacheMeta(
- new PartialPath(fullPath.getDevice(), measurementSchema.getMeasurementId()),
- measurementMNode,
- true);
} else {
cacheMeta(fullPath, measurementMNode, true);
}
- return measurementMNode.getDataType(fullPath.getMeasurement());
+ System.out.println(measurementSchema);
+ return measurementMNode.getDataType(measurement);
} else {
throw e;
}
@@ -974,7 +975,7 @@ public class CMManager extends MManager {
throws MetadataException {
List<PartialPath> result = new ArrayList<>();
// split the paths by the data group they belong to
- Map<PartitionGroup, List<String>> groupPathMap = new HashMap<>();
+ Map<PartitionGroup, List<String>> remoteGroupPathMap = new HashMap<>();
for (Entry<String, String> sgPathEntry : sgPathMap.entrySet()) {
String storageGroupName = sgPathEntry.getKey();
PartialPath pathUnderSG = new PartialPath(sgPathEntry.getValue());
@@ -1000,14 +1001,15 @@ public class CMManager extends MManager {
result.addAll(allTimeseriesName);
} else {
// batch the queries of the same group to reduce communication
- groupPathMap
+ remoteGroupPathMap
.computeIfAbsent(partitionGroup, p -> new ArrayList<>())
.add(pathUnderSG.getFullPath());
}
}
// query each data group separately
- for (Entry<PartitionGroup, List<String>> partitionGroupPathEntry : groupPathMap.entrySet()) {
+ for (Entry<PartitionGroup, List<String>> partitionGroupPathEntry :
+ remoteGroupPathMap.entrySet()) {
PartitionGroup partitionGroup = partitionGroupPathEntry.getKey();
List<String> pathsToQuery = partitionGroupPathEntry.getValue();
result.addAll(getMatchedPaths(partitionGroup, pathsToQuery, withAlias));
@@ -1091,11 +1093,15 @@ public class CMManager extends MManager {
List<PartialPath> partialPaths = new ArrayList<>();
for (int i = 0; i < result.paths.size(); i++) {
try {
- PartialPath partialPath = new PartialPath(result.paths.get(i));
+ PartialPath matchedPath = new PartialPath(result.paths.get(i));
+ if (result.isVectorPath.get(i)) {
+ matchedPath =
+ new VectorPartialPath(matchedPath.getDevice(), matchedPath.getMeasurement());
+ }
if (withAlias) {
- partialPath.setMeasurementAlias(result.aliasList.get(i));
+ matchedPath.setMeasurementAlias(result.aliasList.get(i));
}
- partialPaths.add(partialPath);
+ partialPaths.add(matchedPath);
} catch (IllegalPathException e) {
// ignore
}
@@ -1713,17 +1719,17 @@ public class CMManager extends MManager {
public GetAllPathsResult getAllPaths(List<String> paths, boolean withAlias)
throws MetadataException {
List<String> retPaths = new ArrayList<>();
+ List<Boolean> isVectorPath = new ArrayList<>();
List<String> alias = null;
- if (withAlias) {
- alias = new ArrayList<>();
- }
if (withAlias) {
+ alias = new ArrayList<>();
for (String path : paths) {
List<PartialPath> allTimeseriesPathWithAlias =
super.getAllTimeseriesPathWithAlias(new PartialPath(path), -1, -1).left;
for (PartialPath timeseriesPathWithAlias : allTimeseriesPathWithAlias) {
- retPaths.add(timeseriesPathWithAlias.getFullPath());
+ retPaths.add(timeseriesPathWithAlias.getExactFullPath());
+ isVectorPath.add(timeseriesPathWithAlias instanceof VectorPartialPath);
alias.add(timeseriesPathWithAlias.getMeasurementAlias());
}
}
@@ -1733,6 +1739,7 @@ public class CMManager extends MManager {
GetAllPathsResult getAllPathsResult = new GetAllPathsResult();
getAllPathsResult.setPaths(retPaths);
+ getAllPathsResult.setIsVectorPath(isVectorPath);
getAllPathsResult.setAliasList(alias);
return getAllPathsResult;
}
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/client/sync/SyncClientAdaptorTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/client/sync/SyncClientAdaptorTest.java
index 7c11fce..0871eff 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/client/sync/SyncClientAdaptorTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/client/sync/SyncClientAdaptorTest.java
@@ -245,7 +245,9 @@ public class SyncClientAdaptorTest {
List<String> path,
boolean withAlias,
AsyncMethodCallback<GetAllPathsResult> resultHandler) {
- resultHandler.onComplete(new GetAllPathsResult(path));
+ List<Boolean> isVectorPaths = new ArrayList<>(path.size());
+ Collections.fill(isVectorPaths, false);
+ resultHandler.onComplete(new GetAllPathsResult(path, isVectorPaths));
}
@Override
diff --git a/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java b/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java
index bbfd4c7..598868f 100644
--- a/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/AlignedTimeseriesSessionExample.java
@@ -61,26 +61,17 @@ public class AlignedTimeseriesSessionExample {
createTemplate();
createAlignedTimeseries();
- insertAlignedRecord();
- insertAlignedRecords();
- insertAlignedRecordsOfOneDevices();
-
- insertAlignedStringRecord();
- insertAlignedStringRecords();
-
insertTabletWithAlignedTimeseriesMethod1();
insertTabletWithAlignedTimeseriesMethod2();
- insertNullableTabletWithAlignedTimeseries();
- insertTabletsWithAlignedTimeseries();
-
- selectTest();
- selectWithValueFilterTest();
- selectWithGroupByTest();
- selectWithLastTest();
-
- selectWithAggregationTest();
- selectWithAlignByDeviceTest();
+ // selectTest();
+ // selectWithValueFilterTest();
+ // selectWithGroupByTest();
+ // selectWithLastTest();
+ //
+ // selectWithAggregationTest();
+ //
+ // selectWithAlignByDeviceTest();
session.close();
}
diff --git a/thrift-cluster/src/main/thrift/cluster.thrift b/thrift-cluster/src/main/thrift/cluster.thrift
index 61ae467..755b806 100644
--- a/thrift-cluster/src/main/thrift/cluster.thrift
+++ b/thrift-cluster/src/main/thrift/cluster.thrift
@@ -265,6 +265,7 @@ struct LastQueryRequest {
struct GetAllPathsResult {
1: required list<string> paths
2: optional list<string> aliasList
+ 3: required list<bool> isVectorPath
}
[iotdb] 02/03: Merge branch 'master' into clusterVector
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch clusterVector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 69f2d965c180b7cd8bb9b094aa660c9e1170391f
Merge: 334209f 9579b58
Author: Alima777 <wx...@gmail.com>
AuthorDate: Fri Oct 15 17:11:22 2021 +0800
Merge branch 'master' into clusterVector
.github/dependabot.yml | 30 ++
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 6 +-
cli/pom.xml | 2 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 67 +++++
client-cpp/pom.xml | 2 +-
.../iotdb/cluster/log/StableEntryManager.java | 6 +
.../cluster/log/manage/CommittedEntryManager.java | 12 +
.../manage/FilePartitionedSnapshotLogManager.java | 12 +-
.../iotdb/cluster/log/manage/RaftLogManager.java | 6 +-
.../serializable/SyncLogDequeSerializer.java | 25 ++
.../cluster/server/member/DataGroupMember.java | 2 +-
compile-tools/thrift/pom.xml | 2 +-
.../UserGuide/Ecosystem Integration/Spark IoTDB.md | 18 +-
.../System-Tools/Performance-Tracing-Tool.md | 56 ++--
docs/zh/UserGuide/Appendix/Config-Manual.md | 18 --
.../UserGuide/Ecosystem Integration/Spark IoTDB.md | 18 +-
.../System-Tools/Performance-Tracing-Tool.md | 58 ++--
.../main/java/org/apache/iotdb/JDBCExample.java | 6 +-
.../org/apache/iotdb/PrepareStatementDemo.java | 2 +-
example/trigger/pom.xml | 2 +-
example/udf/pom.xml | 2 +-
.../flink/tsfile/RowTsFileInputFormatTest.java | 14 +-
hive-connector/pom.xml | 2 +-
influxdb-protocol/pom.xml | 4 +-
.../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 22 ++
.../main/java/org/apache/iotdb/jdbc/Constant.java | 12 +
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 6 +-
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 25 ++
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 11 +
.../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 6 +
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 4 +
.../org/apache/iotdb/jdbc/IoTDBTracingInfo.java | 103 +++++++
pom.xml | 18 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 -
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 4 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 -
.../org/apache/iotdb/db/engine/StorageEngine.java | 2 +-
.../engine/storagegroup/StorageGroupProcessor.java | 1 -
.../virtualSg/VirtualStorageGroupManager.java | 2 +-
.../iotdb/db/metadata/logfile/MLogUpgrader.java | 290 ++++++++++++++++++
.../iotdb/db/metadata/logfile/MLogWriter.java | 200 +-----------
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 18 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 13 +
.../iotdb/db/qp/physical/crud/AggregationPlan.java | 42 +--
.../iotdb/db/qp/physical/crud/QueryPlan.java | 13 +
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 1 +
.../iotdb/db/query/context/QueryContext.java | 9 +
.../db/query/control/QueryResourceManager.java | 34 +--
.../apache/iotdb/db/query/control/TracingInfo.java | 85 ------
.../iotdb/db/query/control/TracingManager.java | 229 --------------
.../db/query/control/tracing/TracingConstant.java | 17 +-
.../db/query/control/tracing/TracingInfo.java | 156 ++++++++++
.../db/query/control/tracing/TracingManager.java | 82 +++++
.../db/query/dataset/AlignByDeviceDataSet.java | 4 +-
.../dataset/groupby/GroupByEngineDataSet.java | 6 +
.../query/dataset/groupby/GroupByTimeDataSet.java | 42 ++-
.../groupby/GroupByWithValueFilterDataSet.java | 26 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 25 +-
.../db/query/executor/AggregationExecutor.java | 31 +-
.../db/query/executor/RawDataQueryExecutor.java | 1 +
.../iotdb/db/query/reader/series/SeriesReader.java | 24 +-
.../reader/universal/PriorityMergeReader.java | 13 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 -
.../org/apache/iotdb/db/service/TSServiceImpl.java | 82 +++--
.../org/apache/iotdb/db/utils/AggregateUtils.java | 123 +-------
.../db/integration/IoTDBContinuousQueryIT.java | 38 ++-
.../db/integration/IoTDBRemovePartitionIT.java | 4 +-
.../iotdb/db/integration/IoTDBTracingIT.java | 61 ++--
.../db/integration/IoTDBTriggerExecutionIT.java | 154 ++++++----
.../aggregation/IoTDBAggregationByLevelIT.java | 16 +-
.../iotdb/db/metadata/mlog/MLogUpgraderTest.java | 104 +++++++
.../iotdb/db/query/control/TracingManagerTest.java | 115 -------
.../apache/iotdb/db/utils/EnvironmentUtils.java | 7 +-
.../main/java/org/apache/iotdb/session/Config.java | 4 +
.../org/apache/iotdb/session/InsertConsumer.java | 16 +-
.../java/org/apache/iotdb/session/Session.java | 237 +++++++--------
.../apache/iotdb/session/SessionConnection.java | 1 +
.../org/apache/iotdb/session/pool/SessionPool.java | 37 ++-
.../iotdb/session/{ => util}/SessionUtils.java | 2 +-
.../org/apache/iotdb/session/util/ThreadUtils.java | 45 +++
.../apache/iotdb/session/pool/SessionPoolTest.java | 27 +-
.../apache/iotdb/session/util/ThreadUtilsTest.java | 22 +-
site/pom.xml | 12 +-
spark-iotdb-connector/Readme.md | 168 -----------
spark-tsfile/README.md | 335 ---------------------
thrift/rpc-changelist.md | 22 ++
thrift/src/main/thrift/rpc.thrift | 15 +
zeppelin-interpreter/pom.xml | 2 +-
88 files changed, 1749 insertions(+), 1872 deletions(-)