You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2023/06/15 03:45:42 UTC
[shardingsphere] branch master updated: Implement unload single table & show unloaded single tables; (#26359)
This is an automated email from the ASF dual-hosted git repository.
zhaojinchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 343c4c59432 Implement unload single table & show unloaded single tables; (#26359)
343c4c59432 is described below
commit 343c4c5943207104d0fd04206bd7146e9279d60b
Author: Raigor <ra...@gmail.com>
AuthorDate: Thu Jun 15 11:45:26 2023 +0800
Implement unload single table & show unloaded single tables; (#26359)
* Just load single tables which configured or feature required.
* Implement load single table statement, add single rule for test.
* Fixes #26315, adjust rule order for sharding, single and shadow.
* Optimize loading of single tables
* Update comment.
* Update yaml configuration for test.
* Update yaml configuration for test.
* Load all single tables in pipeline E2E
* Load all single tables in pipeline E2E
* Load all single tables in pipeline E2E
* Implement unload single table & show unloaded single tables;
* Fix format.
* Fix format.
---
.../shardingsphere/infra/datanode/DataNode.java | 10 ++
.../single/datanode/SingleTableDataNodeLoader.java | 13 ++-
.../query/ShowUnloadedSingleTableExecutor.java | 73 +++++++++++++
.../update/UnloadSingleTableStatementUpdater.java | 120 +++++++++++++++++++++
...hardingsphere.distsql.handler.query.RQLExecutor | 1 +
...re.distsql.handler.update.RuleDefinitionUpdater | 1 +
.../src/main/antlr4/imports/single/Keyword.g4 | 4 +
.../src/main/antlr4/imports/single/RDLStatement.g4 | 8 +-
.../parser/core/SingleDistSQLStatementVisitor.java | 7 +-
.../statement/rdl/UnloadSingleTableStatement.java | 5 +-
10 files changed, 235 insertions(+), 7 deletions(-)
diff --git a/infra/common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java b/infra/common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java
index b67674ed71b..319068d3f72 100644
--- a/infra/common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java
+++ b/infra/common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java
@@ -109,6 +109,16 @@ public final class DataNode {
return dataSourceName + DELIMITER + tableName;
}
+ /**
+ * Format data node as string.
+ *
+ * @param databaseType database type
+ * @return formatted data node
+ */
+ public String format(final DatabaseType databaseType) {
+ return databaseType instanceof SchemaSupportedDatabaseType ? dataSourceName + DELIMITER + schemaName + DELIMITER + tableName : dataSourceName + DELIMITER + tableName;
+ }
+
@Override
public boolean equals(final Object object) {
if (this == object) {
diff --git a/kernel/single/core/src/main/java/org/apache/shardingsphere/single/datanode/SingleTableDataNodeLoader.java b/kernel/single/core/src/main/java/org/apache/shardingsphere/single/datanode/SingleTableDataNodeLoader.java
index 08c81457a30..8d913cafe97 100644
--- a/kernel/single/core/src/main/java/org/apache/shardingsphere/single/datanode/SingleTableDataNodeLoader.java
+++ b/kernel/single/core/src/main/java/org/apache/shardingsphere/single/datanode/SingleTableDataNodeLoader.java
@@ -67,8 +67,17 @@ public final class SingleTableDataNodeLoader {
return loadSpecifiedDataNodes(actualDataNodes, featureRequiredSingleTables, configuredTableMap);
}
- private static Map<String, Collection<DataNode>> load(final String databaseName, final DatabaseType databaseType,
- final Map<String, DataSource> dataSourceMap, final Collection<String> excludedTables) {
+ /**
+ * Load single table data nodes.
+ *
+ * @param databaseName database name
+ * @param databaseType database type
+ * @param dataSourceMap data source map
+ * @param excludedTables excluded tables
+ * @return single table data node map
+ */
+ public static Map<String, Collection<DataNode>> load(final String databaseName, final DatabaseType databaseType,
+ final Map<String, DataSource> dataSourceMap, final Collection<String> excludedTables) {
Map<String, Collection<DataNode>> result = new ConcurrentHashMap<>();
for (Entry<String, DataSource> entry : dataSourceMap.entrySet()) {
Map<String, Collection<DataNode>> dataNodeMap = load(databaseName, databaseType, entry.getKey(), entry.getValue(), excludedTables);
diff --git a/kernel/single/distsql/handler/src/main/java/org/apache/shardingsphere/single/distsql/handler/query/ShowUnloadedSingleTableExecutor.java b/kernel/single/distsql/handler/src/main/java/org/apache/shardingsphere/single/distsql/handler/query/ShowUnloadedSingleTableExecutor.java
new file mode 100644
index 00000000000..5d5890d898c
--- /dev/null
+++ b/kernel/single/distsql/handler/src/main/java/org/apache/shardingsphere/single/distsql/handler/query/ShowUnloadedSingleTableExecutor.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.single.distsql.handler.query;
+
+import org.apache.shardingsphere.distsql.handler.query.RQLExecutor;
+import org.apache.shardingsphere.infra.datanode.DataNode;
+import org.apache.shardingsphere.infra.merge.result.impl.local.LocalDataQueryResultRow;
+import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase;
+import org.apache.shardingsphere.infra.metadata.database.resource.ShardingSphereResourceMetaData;
+import org.apache.shardingsphere.single.datanode.SingleTableDataNodeLoader;
+import org.apache.shardingsphere.single.distsql.statement.rql.ShowUnloadedSingleTableStatement;
+import org.apache.shardingsphere.single.rule.SingleRule;
+import org.apache.shardingsphere.single.util.SingleTableLoadUtils;
+
+import javax.sql.DataSource;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Show unloaded single table executor.
+ */
+public final class ShowUnloadedSingleTableExecutor implements RQLExecutor<ShowUnloadedSingleTableStatement> {
+
+ @Override
+ public Collection<String> getColumnNames() {
+ return Arrays.asList("table_name", "storage_unit_name");
+ }
+
+ @Override
+ public Collection<LocalDataQueryResultRow> getRows(final ShardingSphereDatabase database, final ShowUnloadedSingleTableStatement sqlStatement) {
+ Collection<LocalDataQueryResultRow> result = new LinkedList<>();
+ Map<String, Collection<DataNode>> actualDataNodes = getActualDataNodes(database);
+ Optional<SingleRule> singleRule = database.getRuleMetaData().findSingleRule(SingleRule.class);
+ if (singleRule.isPresent()) {
+ Collection<String> singleTableNames = singleRule.get().getLogicTableMapper().getTableNames();
+ for (String each : singleTableNames) {
+ actualDataNodes.remove(each);
+ }
+ }
+ actualDataNodes.forEach((key, value) -> result.add(new LocalDataQueryResultRow(key, value.iterator().next().getDataSourceName())));
+ return result;
+ }
+
+ private Map<String, Collection<DataNode>> getActualDataNodes(final ShardingSphereDatabase database) {
+ ShardingSphereResourceMetaData resourceMetaData = database.getResourceMetaData();
+ Map<String, DataSource> aggregateDataSourceMap = SingleTableLoadUtils.getAggregatedDataSourceMap(resourceMetaData.getDataSources(), database.getRuleMetaData().getRules());
+ Collection<String> excludedTables = SingleTableLoadUtils.getExcludedTables(database.getRuleMetaData().getRules());
+ return SingleTableDataNodeLoader.load(database.getName(), database.getProtocolType(), aggregateDataSourceMap, excludedTables);
+ }
+
+ @Override
+ public String getType() {
+ return ShowUnloadedSingleTableStatement.class.getName();
+ }
+}
diff --git a/kernel/single/distsql/handler/src/main/java/org/apache/shardingsphere/single/distsql/handler/update/UnloadSingleTableStatementUpdater.java b/kernel/single/distsql/handler/src/main/java/org/apache/shardingsphere/single/distsql/handler/update/UnloadSingleTableStatementUpdater.java
new file mode 100644
index 00000000000..7eac0a535c7
--- /dev/null
+++ b/kernel/single/distsql/handler/src/main/java/org/apache/shardingsphere/single/distsql/handler/update/UnloadSingleTableStatementUpdater.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.single.distsql.handler.update;
+
+import com.google.common.base.Splitter;
+import org.apache.shardingsphere.dialect.exception.syntax.table.NoSuchTableException;
+import org.apache.shardingsphere.distsql.handler.exception.rule.MissingRequiredRuleException;
+import org.apache.shardingsphere.distsql.handler.update.RuleDefinitionAlterUpdater;
+import org.apache.shardingsphere.infra.database.type.DatabaseType;
+import org.apache.shardingsphere.infra.database.type.DatabaseTypeEngine;
+import org.apache.shardingsphere.infra.datanode.DataNode;
+import org.apache.shardingsphere.infra.metadata.database.ShardingSphereDatabase;
+import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereTable;
+import org.apache.shardingsphere.infra.util.exception.ShardingSpherePreconditions;
+import org.apache.shardingsphere.single.api.config.SingleRuleConfiguration;
+import org.apache.shardingsphere.single.constant.SingleTableConstants;
+import org.apache.shardingsphere.single.distsql.statement.rdl.UnloadSingleTableStatement;
+import org.apache.shardingsphere.single.exception.SingleTableNotFoundException;
+import org.apache.shardingsphere.single.rule.SingleRule;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Unload single table statement updater.
+ */
+public final class UnloadSingleTableStatementUpdater implements RuleDefinitionAlterUpdater<UnloadSingleTableStatement, SingleRuleConfiguration> {
+
+ @Override
+ public void checkSQLStatement(final ShardingSphereDatabase database, final UnloadSingleTableStatement sqlStatement, final SingleRuleConfiguration currentRuleConfig) {
+ checkCurrentRuleConfig(database.getName(), currentRuleConfig);
+ checkTables(database, sqlStatement, currentRuleConfig);
+ }
+
+ private void checkCurrentRuleConfig(final String databaseName, final SingleRuleConfiguration currentRuleConfig) {
+ ShardingSpherePreconditions.checkState(null != currentRuleConfig, () -> new MissingRequiredRuleException("Single", databaseName));
+ }
+
+ private void checkTables(final ShardingSphereDatabase database, final UnloadSingleTableStatement sqlStatement, final SingleRuleConfiguration currentRuleConfig) {
+ if (sqlStatement.isUnloadAllTables()) {
+ return;
+ }
+ Collection<String> allTables = getAllTableNames(database);
+ SingleRule singleRule = database.getRuleMetaData().getSingleRule(SingleRule.class);
+ Collection<String> singleTables = singleRule.getLogicTableMapper().getTableNames();
+ for (String each : sqlStatement.getTables()) {
+ checkTableExist(allTables, each);
+ checkIsSingleTable(singleTables, each);
+ checkTableRuleExist(database.getName(), database.getProtocolType(), currentRuleConfig, singleRule.getDataNodesByTableName(each), each);
+ }
+ }
+
+ private void checkTableExist(final Collection<String> allTables, final String tableName) {
+ ShardingSpherePreconditions.checkState(allTables.contains(tableName), () -> new NoSuchTableException(tableName));
+ }
+
+ private void checkIsSingleTable(final Collection<String> singleTables, final String tableName) {
+ ShardingSpherePreconditions.checkState(singleTables.contains(tableName), () -> new SingleTableNotFoundException(tableName));
+ }
+
+ private Collection<String> getAllTableNames(final ShardingSphereDatabase database) {
+ String defaultSchemaName = DatabaseTypeEngine.getDefaultSchemaName(database.getProtocolType(), database.getName());
+ return database.getSchema(defaultSchemaName).getTables().values().stream().map(ShardingSphereTable::getName).collect(Collectors.toList());
+ }
+
+ private void checkTableRuleExist(final String databaseName, final DatabaseType databaseType, final SingleRuleConfiguration currentRuleConfig,
+ final Collection<DataNode> dataNodes, final String tableName) {
+ ShardingSpherePreconditions.checkState(!dataNodes.isEmpty(), () -> new MissingRequiredRuleException("Single", databaseName, tableName));
+ DataNode dataNode = dataNodes.iterator().next();
+ ShardingSpherePreconditions.checkState(currentRuleConfig.getTables().contains(dataNode.format(databaseType)), () -> new MissingRequiredRuleException("Single", databaseName, tableName));
+ }
+
+ @Override
+ public SingleRuleConfiguration buildToBeAlteredRuleConfiguration(final UnloadSingleTableStatement sqlStatement) {
+ SingleRuleConfiguration result = new SingleRuleConfiguration();
+ result.getTables().addAll(sqlStatement.isUnloadAllTables() ? Collections.singletonList(SingleTableConstants.ASTERISK) : sqlStatement.getTables());
+ return result;
+ }
+
+ @Override
+ public void updateCurrentRuleConfiguration(final SingleRuleConfiguration currentRuleConfig, final SingleRuleConfiguration toBeAlteredRuleConfig) {
+ if (toBeAlteredRuleConfig.getTables().contains(SingleTableConstants.ASTERISK)) {
+ currentRuleConfig.getTables().clear();
+ } else {
+ currentRuleConfig.getTables().removeIf(each -> toBeAlteredRuleConfig.getTables().contains(extractTableName(each)));
+ }
+ }
+
+ private String extractTableName(final String tableNode) {
+ List<String> segments = Splitter.on(".").trimResults().splitToList(tableNode);
+ return segments.get(segments.size() - 1);
+ }
+
+ @Override
+ public Class<SingleRuleConfiguration> getRuleConfigurationClass() {
+ return SingleRuleConfiguration.class;
+ }
+
+ @Override
+ public String getType() {
+ return UnloadSingleTableStatement.class.getName();
+ }
+}
diff --git a/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.query.RQLExecutor b/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.query.RQLExecutor
index 28f9b11ee6b..433d8f9a21a 100644
--- a/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.query.RQLExecutor
+++ b/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.query.RQLExecutor
@@ -18,3 +18,4 @@
org.apache.shardingsphere.single.distsql.handler.query.ShowSingleTableExecutor
org.apache.shardingsphere.single.distsql.handler.query.ShowDefaultSingleTableStorageUnitExecutor
org.apache.shardingsphere.single.distsql.handler.query.CountSingleTableExecutor
+org.apache.shardingsphere.single.distsql.handler.query.ShowUnloadedSingleTableExecutor
diff --git a/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.update.RuleDefinitionUpdater b/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.update.RuleDefinitionUpdater
index ee8080b7053..fb7371324c1 100644
--- a/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.update.RuleDefinitionUpdater
+++ b/kernel/single/distsql/handler/src/main/resources/META-INF/services/org.apache.shardingsphere.distsql.handler.update.RuleDefinitionUpdater
@@ -17,3 +17,4 @@
org.apache.shardingsphere.single.distsql.handler.update.SetDefaultSingleTableStorageUnitStatementUpdater
org.apache.shardingsphere.single.distsql.handler.update.LoadSingleTableStatementUpdater
+org.apache.shardingsphere.single.distsql.handler.update.UnloadSingleTableStatementUpdater
diff --git a/kernel/single/distsql/parser/src/main/antlr4/imports/single/Keyword.g4 b/kernel/single/distsql/parser/src/main/antlr4/imports/single/Keyword.g4
index 4f8a8f634df..f7d1fb4a728 100644
--- a/kernel/single/distsql/parser/src/main/antlr4/imports/single/Keyword.g4
+++ b/kernel/single/distsql/parser/src/main/antlr4/imports/single/Keyword.g4
@@ -47,6 +47,10 @@ TABLES
: T A B L E S
;
+ALL
+ : A L L
+ ;
+
STORAGE
: S T O R A G E
;
diff --git a/kernel/single/distsql/parser/src/main/antlr4/imports/single/RDLStatement.g4 b/kernel/single/distsql/parser/src/main/antlr4/imports/single/RDLStatement.g4
index 76ed6bd9a79..4a414eb91b4 100644
--- a/kernel/single/distsql/parser/src/main/antlr4/imports/single/RDLStatement.g4
+++ b/kernel/single/distsql/parser/src/main/antlr4/imports/single/RDLStatement.g4
@@ -28,13 +28,19 @@ loadSingleTable
;
unloadSingleTable
- : UNLOAD SINGLE TABLE tableDefinition
+ : UNLOAD SINGLE TABLE tableNames
+ | UNLOAD SINGLE TABLE ASTERISK_
+ | UNLOAD ALL SINGLE TABLES
;
tableDefinition
: tableIdentifier (COMMA_ tableIdentifier)*
;
+tableNames
+ : tableName (COMMA_ tableName)*
+ ;
+
tableIdentifier
: ASTERISK_ DOTASTERISK_ # allTables
| ASTERISK_ DOTASTERISK_ DOTASTERISK_ # allTablesWithSchema
diff --git a/kernel/single/distsql/parser/src/main/java/org/apache/shardingsphere/single/distsql/parser/core/SingleDistSQLStatementVisitor.java b/kernel/single/distsql/parser/src/main/java/org/apache/shardingsphere/single/distsql/parser/core/SingleDistSQLStatementVisitor.java
index f1373ef7b8f..aef1067c271 100644
--- a/kernel/single/distsql/parser/src/main/java/org/apache/shardingsphere/single/distsql/parser/core/SingleDistSQLStatementVisitor.java
+++ b/kernel/single/distsql/parser/src/main/java/org/apache/shardingsphere/single/distsql/parser/core/SingleDistSQLStatementVisitor.java
@@ -49,6 +49,7 @@ import org.apache.shardingsphere.sql.parser.sql.common.segment.generic.DatabaseS
import org.apache.shardingsphere.sql.parser.sql.common.value.identifier.IdentifierValue;
import java.util.Collection;
+import java.util.Collections;
import java.util.stream.Collectors;
/**
@@ -85,8 +86,10 @@ public final class SingleDistSQLStatementVisitor extends SingleDistSQLStatementB
@Override
public ASTNode visitUnloadSingleTable(final UnloadSingleTableContext ctx) {
- Collection<SingleTableSegment> tables = ctx.tableDefinition().tableIdentifier().stream().map(this::getSingleTableSegment).collect(Collectors.toSet());
- return new UnloadSingleTableStatement(tables);
+ if (null != ctx.ALL() || null != ctx.ASTERISK_()) {
+ return new UnloadSingleTableStatement(true, Collections.emptyList());
+ }
+ return new UnloadSingleTableStatement(false, ctx.tableNames().tableName().stream().map(this::getIdentifierValue).collect(Collectors.toSet()));
}
private SingleTableSegment getSingleTableSegment(final TableIdentifierContext ctx) {
diff --git a/kernel/single/distsql/statement/src/main/java/org/apache/shardingsphere/single/distsql/statement/rdl/UnloadSingleTableStatement.java b/kernel/single/distsql/statement/src/main/java/org/apache/shardingsphere/single/distsql/statement/rdl/UnloadSingleTableStatement.java
index 5a9142265df..69ca6a98897 100644
--- a/kernel/single/distsql/statement/src/main/java/org/apache/shardingsphere/single/distsql/statement/rdl/UnloadSingleTableStatement.java
+++ b/kernel/single/distsql/statement/src/main/java/org/apache/shardingsphere/single/distsql/statement/rdl/UnloadSingleTableStatement.java
@@ -20,7 +20,6 @@ package org.apache.shardingsphere.single.distsql.statement.rdl;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.apache.shardingsphere.distsql.parser.statement.rdl.alter.AlterRuleStatement;
-import org.apache.shardingsphere.single.distsql.segment.SingleTableSegment;
import java.util.Collection;
@@ -31,5 +30,7 @@ import java.util.Collection;
@Getter
public final class UnloadSingleTableStatement extends AlterRuleStatement {
- private final Collection<SingleTableSegment> tables;
+ private final boolean isUnloadAllTables;
+
+ private final Collection<String> tables;
}