You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/04/29 03:09:31 UTC
[iotdb] branch master updated: [IOTDB-3027] Feature/count storage group (#5728)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5eb2308198 [IOTDB-3027] Feature/count storage group (#5728)
5eb2308198 is described below
commit 5eb23081981584a38bc8465663187a588b78823d
Author: ZhangHongYin <46...@users.noreply.github.com>
AuthorDate: Fri Apr 29 11:09:27 2022 +0800
[IOTDB-3027] Feature/count storage group (#5728)
---
.../apache/iotdb/db/client/ConfigNodeClient.java | 17 ++++
.../iotdb/db/mpp/common/header/HeaderConstant.java | 10 +--
.../db/mpp/execution/config/ConfigTaskVisitor.java | 7 ++
.../execution/config/CountStorageGroupTask.java | 97 ++++++++++++++++++++++
.../apache/iotdb/db/mpp/sql/analyze/Analyzer.java | 10 +++
.../apache/iotdb/db/mpp/sql/parser/ASTVisitor.java | 4 +-
.../db/mpp/sql/statement/StatementVisitor.java | 6 ++
.../metadata/CountStorageGroupStatement.java | 16 +++-
8 files changed, 154 insertions(+), 13 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
index 15bdd581af..9069fe2d67 100644
--- a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
+++ b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.utils.CommonUtils;
import org.apache.iotdb.confignode.rpc.thrift.ConfigIService;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq;
import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp;
+import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeLocationResp;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp;
@@ -223,6 +224,22 @@ public class ConfigNodeClient {
throw new IoTDBConnectionException(MSG_RECONNECTION_FAIL);
}
+ public TCountStorageGroupResp countMatchedStorageGroups(List<String> storageGroupPathPattern)
+ throws IoTDBConnectionException {
+ for (int i = 0; i < RETRY_NUM; i++) {
+ try {
+ TCountStorageGroupResp resp = client.countMatchedStorageGroups(storageGroupPathPattern);
+ if (!updateConfigNodeLeader(resp.status)) {
+ return resp;
+ }
+ } catch (TException e) {
+ configLeader = null;
+ }
+ reconnect();
+ }
+ throw new IoTDBConnectionException(MSG_RECONNECTION_FAIL);
+ }
+
public TStorageGroupSchemaResp getMatchedStorageGroupSchemas(List<String> storageGroupPathPattern)
throws IoTDBConnectionException {
for (int i = 0; i < RETRY_NUM; i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java
index f4bac2051e..8f3eff26ba 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java
@@ -40,6 +40,7 @@ public class HeaderConstant {
public static final String COLUMN_TAGS = "tags";
public static final String COLUMN_ATTRIBUTES = "attributes";
public static final String COLUMN_IS_ALIGNED = "isAligned";
+ public static final String COLUMN_COUNT = "count";
// column names for count statement
public static final String COLUMN_COLUMN = "column";
@@ -100,18 +101,12 @@ public class HeaderConstant {
new ColumnHeader(COLUMN_TAGS, TSDataType.TEXT),
new ColumnHeader(COLUMN_ATTRIBUTES, TSDataType.TEXT)),
true);
- }
-
- static {
showDevicesHeader =
new DatasetHeader(
Arrays.asList(
new ColumnHeader(COLUMN_DEVICES, TSDataType.TEXT),
new ColumnHeader(COLUMN_IS_ALIGNED, TSDataType.TEXT)),
true);
- }
-
- static {
showDevicesWithSgHeader =
new DatasetHeader(
Arrays.asList(
@@ -119,9 +114,6 @@ public class HeaderConstant {
new ColumnHeader(COLUMN_STORAGE_GROUP, TSDataType.TEXT),
new ColumnHeader(COLUMN_IS_ALIGNED, TSDataType.TEXT)),
true);
- }
-
- static {
showStorageGroupHeader =
new DatasetHeader(
Collections.singletonList(new ColumnHeader(COLUMN_STORAGE_GROUP, TSDataType.TEXT)),
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/config/ConfigTaskVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/config/ConfigTaskVisitor.java
index 7c4a944f12..0910575174 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/config/ConfigTaskVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/config/ConfigTaskVisitor.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.mpp.execution.config;
import org.apache.iotdb.db.mpp.sql.statement.Statement;
import org.apache.iotdb.db.mpp.sql.statement.StatementNode;
import org.apache.iotdb.db.mpp.sql.statement.StatementVisitor;
+import org.apache.iotdb.db.mpp.sql.statement.metadata.CountStorageGroupStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.SetStorageGroupStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.ShowStorageGroupStatement;
import org.apache.iotdb.db.mpp.sql.statement.sys.AuthorStatement;
@@ -52,6 +53,12 @@ public class ConfigTaskVisitor
return new ShowStorageGroupTask(statement);
}
+ @Override
+ public IConfigTask visitCountStorageGroup(
+ CountStorageGroupStatement statement, TaskContext context) {
+ return new CountStorageGroupTask(statement);
+ }
+
@Override
public IConfigTask visitAuthor(AuthorStatement statement, TaskContext context) {
return new AuthorizerConfigTask(statement);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/config/CountStorageGroupTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/config/CountStorageGroupTask.java
new file mode 100644
index 0000000000..4a5ebf455e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/config/CountStorageGroupTask.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.execution.config;
+
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.BadNodeUrlException;
+import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp;
+import org.apache.iotdb.db.client.ConfigNodeClient;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.localconfignode.LocalConfigNode;
+import org.apache.iotdb.db.mpp.common.header.ColumnHeader;
+import org.apache.iotdb.db.mpp.common.header.DatasetHeader;
+import org.apache.iotdb.db.mpp.sql.statement.metadata.CountStorageGroupStatement;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class CountStorageGroupTask implements IConfigTask {
+ private static final Logger LOGGER = LoggerFactory.getLogger(CountStorageGroupTask.class);
+
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+
+ private CountStorageGroupStatement countStorageGroupStatement;
+
+ public CountStorageGroupTask(CountStorageGroupStatement countStorageGroupStatement) {
+ this.countStorageGroupStatement = countStorageGroupStatement;
+ }
+
+ @Override
+ public ListenableFuture<ConfigTaskResult> execute() throws InterruptedException {
+ SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ int storageGroupNum = 0;
+ if (config.isClusterMode()) {
+ List<String> storageGroupPathPattern =
+ Arrays.asList(countStorageGroupStatement.getPartialPath().getNodes());
+ ConfigNodeClient client = null;
+ try {
+ client = new ConfigNodeClient();
+ TCountStorageGroupResp resp = client.countMatchedStorageGroups(storageGroupPathPattern);
+ storageGroupNum = resp.getCount();
+ } catch (IoTDBConnectionException | BadNodeUrlException e) {
+ LOGGER.error("Failed to connect to config node.");
+ future.setException(e);
+ }
+ } else {
+ try {
+ storageGroupNum =
+ LocalConfigNode.getInstance()
+ .getStorageGroupNum(
+ countStorageGroupStatement.getPartialPath(),
+ countStorageGroupStatement.isPrefixPath());
+ } catch (MetadataException e) {
+ future.setException(e);
+ }
+ }
+ // build TSBlock
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT32));
+ builder.getTimeColumnBuilder().writeLong(0L);
+ builder.getColumnBuilder(0).writeInt(storageGroupNum);
+ builder.declarePosition();
+ ColumnHeader storageGroupColumnHeader =
+ new ColumnHeader(IoTDBConstant.COLUMN_COUNT, TSDataType.INT32);
+ DatasetHeader datasetHeader =
+ new DatasetHeader(Collections.singletonList(storageGroupColumnHeader), true);
+ future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader));
+ return future;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/sql/analyze/Analyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/sql/analyze/Analyzer.java
index c85e5e254a..4599a54dd9 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/analyze/Analyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/analyze/Analyzer.java
@@ -52,6 +52,7 @@ import org.apache.iotdb.db.mpp.sql.statement.crud.QueryStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.AlterTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountDevicesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountLevelTimeSeriesStatement;
+import org.apache.iotdb.db.mpp.sql.statement.metadata.CountStorageGroupStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CreateAlignedTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CreateTimeSeriesStatement;
@@ -452,6 +453,15 @@ public class Analyzer {
return analysis;
}
+ @Override
+ public Analysis visitCountStorageGroup(
+ CountStorageGroupStatement countStorageGroupStatement, MPPQueryContext context) {
+ Analysis analysis = new Analysis();
+ analysis.setStatement(countStorageGroupStatement);
+ analysis.setRespDatasetHeader(HeaderConstant.countStorageGroupHeader);
+ return analysis;
+ }
+
@Override
public Analysis visitSchemaFetch(
SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/sql/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/sql/parser/ASTVisitor.java
index 886aa67f3c..ff33292740 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/parser/ASTVisitor.java
@@ -57,7 +57,7 @@ import org.apache.iotdb.db.mpp.sql.statement.metadata.AlterTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountDevicesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountLevelTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountNodesStatement;
-import org.apache.iotdb.db.mpp.sql.statement.metadata.CountStatement;
+import org.apache.iotdb.db.mpp.sql.statement.metadata.CountStorageGroupStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CreateAlignedTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CreateTimeSeriesStatement;
@@ -494,7 +494,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
} else {
path = new PartialPath(SQLConstant.getSingleRootArray());
}
- return new CountStatement(path);
+ return new CountStorageGroupStatement(path);
}
/** Data Manipulation Language (DML) */
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/StatementVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/StatementVisitor.java
index 6e47a2c928..89ad3a774f 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/StatementVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/StatementVisitor.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.db.mpp.sql.statement.crud.UDTFQueryStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.AlterTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountDevicesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountLevelTimeSeriesStatement;
+import org.apache.iotdb.db.mpp.sql.statement.metadata.CountStorageGroupStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CountTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CreateAlignedTimeSeriesStatement;
import org.apache.iotdb.db.mpp.sql.statement.metadata.CreateTimeSeriesStatement;
@@ -150,6 +151,11 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(showDevicesStatement, context);
}
+ public R visitCountStorageGroup(
+ CountStorageGroupStatement countStorageGroupStatement, C context) {
+ return visitStatement(countStorageGroupStatement, context);
+ }
+
public R visitCountDevices(CountDevicesStatement countStatement, C context) {
return visitStatement(countStatement, context);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/metadata/CountStorageGroupStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/metadata/CountStorageGroupStatement.java
index 26f9fc5111..eb98cba9df 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/metadata/CountStorageGroupStatement.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/metadata/CountStorageGroupStatement.java
@@ -20,11 +20,23 @@
package org.apache.iotdb.db.mpp.sql.statement.metadata;
import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.db.mpp.sql.analyze.QueryType;
+import org.apache.iotdb.db.mpp.sql.statement.IConfigStatement;
+import org.apache.iotdb.db.mpp.sql.statement.StatementVisitor;
-public class CountStorageGroupStatement extends CountStatement {
- private PartialPath partialPath;
+public class CountStorageGroupStatement extends CountStatement implements IConfigStatement {
public CountStorageGroupStatement(PartialPath partialPath) {
super(partialPath);
}
+
+ @Override
+ public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
+ return visitor.visitCountStorageGroup(this, context);
+ }
+
+ @Override
+ public QueryType getQueryType() {
+ return QueryType.READ;
+ }
}