You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by du...@apache.org on 2022/05/25 10:41:22 UTC
[shardingsphere] branch master updated: Fixed reload meta data not refresh and add rebuild single table rule (#17941)
This is an automated email from the ASF dual-hosted git repository.
duanzhengqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 4c25dd2f8cc Fixed reload meta data not refresh and add rebuild single table rule (#17941)
4c25dd2f8cc is described below
commit 4c25dd2f8cc54df1ce2e9beea454a65eb355765a
Author: zhaojinchao <zh...@apache.org>
AuthorDate: Wed May 25 18:41:15 2022 +0800
Fixed reload meta data not refresh and add rebuild single table rule (#17941)
* Fixed reload meta data not refresh and add build rules
* Fix ci
* Fix unit test
* Fix unit test
* Rename method name.
---
.../mode/manager/ContextManager.java | 39 +++++++++++++---------
.../service/SchemaMetaDataPersistService.java | 26 +++++----------
.../mode/manager/ContextManagerTest.java | 6 ++--
.../service/SchemaMetaDataPersistServiceTest.java | 8 +----
.../cluster/ClusterContextManagerBuilder.java | 9 ++---
.../SchemaMetaDataRegistrySubscriber.java | 6 +---
.../SchemaMetaDataRegistrySubscriberTest.java | 4 +--
.../updatable/RefreshTableMetadataHandler.java | 2 +-
8 files changed, 43 insertions(+), 57 deletions(-)
diff --git a/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/ContextManager.java b/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/ContextManager.java
index 11350d6046d..f9461137ec1 100644
--- a/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/ContextManager.java
+++ b/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/ContextManager.java
@@ -70,6 +70,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
@@ -197,13 +198,8 @@ public final class ContextManager implements AutoCloseable {
}
private void persistMetaData(final MetaDataContexts metaDataContexts) {
- metaDataContexts.getDatabaseMap().forEach((databaseName, schemas) -> schemas.getSchemas().forEach((schemaName, tables) -> {
- if (tables.getTables().isEmpty()) {
- metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().persistSchema(databaseName, schemaName));
- } else {
- metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().persistTables(databaseName, schemaName, tables));
- }
- }));
+ metaDataContexts.getDatabaseMap().forEach((databaseName, schemas) -> schemas.getSchemas().forEach((schemaName, tables) ->
+ metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().persistMetaData(databaseName, schemaName, tables))));
}
private void alterTableSchema(final String databaseName, final String schemaName, final TableMetaData changedTableMetaData) {
@@ -395,17 +391,15 @@ public final class ContextManager implements AutoCloseable {
* Reload meta data.
*
* @param databaseName database name to be reloaded
- * @param schemaName schema name to be reloaded
*/
- public void reloadMetaData(final String databaseName, final String schemaName) {
+ public void reloadMetaData(final String databaseName) {
try {
Map<String, ShardingSphereSchema> schemas = loadActualSchema(databaseName);
+ deleteSchemas(databaseName, schemas);
alterSchemas(databaseName, schemas);
- for (ShardingSphereSchema each : schemas.values()) {
- metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().persistTables(databaseName, schemaName, each));
- }
+ persistMetaData(metaDataContexts);
} catch (final SQLException ex) {
- log.error("Reload database:{} meta data of schema:{} failed", databaseName, schemaName, ex);
+ log.error("Reload database:{} failed", databaseName, ex);
}
}
@@ -447,21 +441,34 @@ public final class ContextManager implements AutoCloseable {
}
}
+ private void deleteSchemas(final String databaseName, final Map<String, ShardingSphereSchema> actualSchemas) {
+ Map<String, ShardingSphereSchema> originalSchemas = metaDataContexts.getDatabaseMetaData(databaseName).getSchemas();
+ if (originalSchemas.isEmpty()) {
+ return;
+ }
+ originalSchemas.forEach((key, value) -> {
+ if (null == actualSchemas.get(key)) {
+ metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().deleteSchema(databaseName, key));
+ }
+ });
+ }
+
private void loadTableMetaData(final String databaseName, final String schemaName, final String tableName, final SchemaBuilderMaterials materials) throws SQLException {
SchemaMetaData schemaMetaData = SchemaMetaDataBuilder.load(Collections.singletonList(tableName), materials).getOrDefault(schemaName, new SchemaMetaData("", Collections.emptyMap()));
if (schemaMetaData.getTables().containsKey(tableName)) {
metaDataContexts.getDatabaseMetaData(databaseName).getSchemas().get(schemaName).put(tableName, schemaMetaData.getTables().get(tableName));
metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService()
- .persistTables(databaseName, schemaName, metaDataContexts.getDatabaseMetaData(databaseName).getSchemas().get(schemaName)));
+ .persistMetaData(databaseName, schemaName, metaDataContexts.getDatabaseMetaData(databaseName).getSchemas().get(schemaName)));
}
}
private Map<String, ShardingSphereSchema> loadActualSchema(final String databaseName) throws SQLException {
ShardingSphereDatabase database = metaDataContexts.getDatabaseMetaData(databaseName);
Map<String, DataSource> dataSourceMap = database.getResource().getDataSources();
- Collection<ShardingSphereRule> rules = metaDataContexts.getDatabaseMap().get(databaseName).getRuleMetaData().getRules();
+ refreshRules(databaseName, database);
DatabaseType databaseType = DatabaseTypeEngine.getDatabaseType(dataSourceMap.values());
- Map<String, ShardingSphereSchema> result = SchemaLoader.load(databaseName, database.getProtocolType(), databaseType, dataSourceMap, rules, metaDataContexts.getProps());
+ Map<String, ShardingSphereSchema> result = new ConcurrentHashMap<>();
+ result.putAll(SchemaLoader.load(databaseName, database.getProtocolType(), databaseType, dataSourceMap, database.getRuleMetaData().getRules(), metaDataContexts.getProps()));
result.putAll(SystemSchemaBuilder.build(databaseName, database.getProtocolType()));
return result;
}
diff --git a/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistService.java b/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistService.java
index d81d307bdfa..5a33d3b343f 100644
--- a/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistService.java
+++ b/shardingsphere-mode/shardingsphere-mode-core/src/main/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistService.java
@@ -40,25 +40,26 @@ public final class SchemaMetaDataPersistService {
private final PersistRepository repository;
/**
- * Persist tables.
+ * Persist meta data.
*
* @param databaseName database name to be persisted
* @param schemaName schema name to be persisted
* @param schema schema to be persisted
*/
- public void persistTables(final String databaseName, final String schemaName, final ShardingSphereSchema schema) {
- if (null == schema) {
- return;
- }
+ public void persistMetaData(final String databaseName, final String schemaName, final ShardingSphereSchema schema) {
Optional<ShardingSphereSchema> originalSchema = load(databaseName, schemaName);
if (originalSchema.isPresent()) {
compareAndPersist(databaseName, schemaName, schema, originalSchema.get());
return;
}
- persistTables(databaseName, schemaName, schema.getTables());
+ persistMetaData(databaseName, schemaName, schema.getTables());
}
- private void persistTables(final String databaseName, final String schemaName, final Map<String, TableMetaData> tables) {
+ private void persistMetaData(final String databaseName, final String schemaName, final Map<String, TableMetaData> tables) {
+ if (tables.isEmpty()) {
+ persistSchema(databaseName, schemaName);
+ return;
+ }
tables.forEach((key, value) -> repository.persist(DatabaseMetaDataNode.getTableMetaDataPath(databaseName, schemaName, key),
YamlEngine.marshal(new TableMetaDataYamlSwapper().swapToYamlConfiguration(value))));
}
@@ -75,15 +76,6 @@ public final class SchemaMetaDataPersistService {
YamlEngine.marshal(new TableMetaDataYamlSwapper().swapToYamlConfiguration(tableMetaData)));
}
- /**
- * Persist database.
- *
- * @param databaseName database name
- */
- public void persistDatabase(final String databaseName) {
- repository.persist(DatabaseMetaDataNode.getDatabaseNamePath(databaseName), "");
- }
-
/**
* Persist schema.
*
@@ -108,7 +100,7 @@ public final class SchemaMetaDataPersistService {
}
}
if (!cachedLocalTables.isEmpty()) {
- persistTables(databaseName, schemaName, cachedLocalTables);
+ persistMetaData(databaseName, schemaName, cachedLocalTables);
}
}
diff --git a/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/ContextManagerTest.java b/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/ContextManagerTest.java
index b7f7a4bbe3f..b3a947299fa 100644
--- a/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/ContextManagerTest.java
+++ b/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/ContextManagerTest.java
@@ -80,6 +80,8 @@ public final class ContextManagerTest {
when(metaDataContexts.getProps()).thenReturn(new ConfigurationProperties(new Properties()));
when(metaDataContexts.getDatabaseMetaData("foo_db").getResource().getDatabaseType()).thenReturn(new MySQLDatabaseType());
when(metaDataContexts.getDatabaseMetaData("foo_db").getProtocolType()).thenReturn(new MySQLDatabaseType());
+ when(metaDataContexts.getDatabaseMetaData("foo_db").getSchemas()).thenReturn(Collections.singletonMap("foo_db", new ShardingSphereSchema()));
+ when(metaDataContexts.getDatabaseMetaData("foo_db").getRuleMetaData()).thenReturn(new ShardingSphereRuleMetaData(Collections.emptyList(), Collections.emptyList()));
contextManager = new ContextManager(metaDataContexts, mock(TransactionContexts.class), mock(InstanceContext.class));
}
@@ -252,8 +254,8 @@ public final class ContextManagerTest {
MetaDataPersistService metaDataPersistService = mock(MetaDataPersistService.class);
when(metaDataPersistService.getSchemaMetaDataService()).thenReturn(schemaMetaDataPersistService);
when(metaDataContexts.getPersistService()).thenReturn(Optional.of(metaDataPersistService));
- contextManager.reloadMetaData("foo_db", "foo_schema");
- verify(schemaMetaDataPersistService, times(1)).persistTables(eq("foo_db"), eq("foo_schema"), any(ShardingSphereSchema.class));
+ contextManager.reloadMetaData("foo_db");
+ verify(schemaMetaDataPersistService, times(1)).persistMetaData(eq("foo_db"), eq("foo_db"), any(ShardingSphereSchema.class));
contextManager.reloadMetaData("foo_db", "foo_schema", "foo_table");
assertNotNull(contextManager.getMetaDataContexts().getDatabaseMetaData("foo_db").getSchemas().get("foo_db"));
assertTrue(contextManager.getMetaDataContexts().getDatabaseMetaData("foo_db").getResource().getDataSources().containsKey("foo_ds"));
diff --git a/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistServiceTest.java b/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistServiceTest.java
index 55f60046365..f3b5353ca2b 100644
--- a/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistServiceTest.java
+++ b/shardingsphere-mode/shardingsphere-mode-core/src/test/java/org/apache/shardingsphere/mode/metadata/persist/service/SchemaMetaDataPersistServiceTest.java
@@ -58,16 +58,10 @@ public final class SchemaMetaDataPersistServiceTest {
TableMetaData tableMetaData = new TableMetaDataYamlSwapper().swapToObject(YamlEngine.unmarshal(readYAML(), YamlTableMetaData.class));
ShardingSphereSchema schema = new ShardingSphereSchema();
schema.getTables().put("t_order", tableMetaData);
- new SchemaMetaDataPersistService(repository).persistTables("foo_db", "foo_schema", schema);
+ new SchemaMetaDataPersistService(repository).persistMetaData("foo_db", "foo_schema", schema);
verify(repository).persist(eq("/metadata/foo_db/schemas/foo_schema/tables/t_order"), anyString());
}
- @Test
- public void assertPersistDatabase() {
- new SchemaMetaDataPersistService(repository).persistDatabase("foo_db");
- verify(repository).persist(eq("/metadata/foo_db"), anyString());
- }
-
@Test
public void assertDeleteDatabase() {
new SchemaMetaDataPersistService(repository).deleteDatabase("foo_db");
diff --git a/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/ClusterContextManagerBuilder.java b/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/ClusterContextManagerBuilder.java
index b8038a2ae5c..cccf3677c22 100644
--- a/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/ClusterContextManagerBuilder.java
+++ b/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/ClusterContextManagerBuilder.java
@@ -143,13 +143,8 @@ public final class ClusterContextManagerBuilder implements ContextManagerBuilder
}
private void persistMetaData(final MetaDataContexts metaDataContexts) {
- metaDataContexts.getDatabaseMap().forEach((databaseName, schemas) -> schemas.getSchemas().forEach((schemaName, tables) -> {
- if (tables.getTables().isEmpty()) {
- metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().persistSchema(databaseName, schemaName));
- } else {
- metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().persistTables(databaseName, schemaName, tables));
- }
- }));
+ metaDataContexts.getDatabaseMap().forEach((databaseName, schemas) -> schemas.getSchemas().forEach((schemaName, tables) ->
+ metaDataContexts.getPersistService().ifPresent(optional -> optional.getSchemaMetaDataService().persistMetaData(databaseName, schemaName, tables))));
}
private ContextManager createContextManager(final ClusterPersistRepository repository, final MetaDataPersistService metaDataPersistService,
diff --git a/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegistrySubscriber.java b/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegistrySu [...]
index 2474b651f65..32e72b2db9c 100644
--- a/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegistrySubscriber.java
+++ b/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/main/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegistrySubscriber.java
@@ -66,11 +66,7 @@ public final class SchemaMetaDataRegistrySubscriber {
*/
@Subscribe
public void alterSchema(final AlterSchemaEvent event) {
- if (event.getSchema().getTables().isEmpty()) {
- persistService.persistSchema(event.getDatabaseName(), event.getRenameSchemaName());
- } else {
- persistService.persistTables(event.getDatabaseName(), event.getRenameSchemaName(), event.getSchema());
- }
+ persistService.persistMetaData(event.getDatabaseName(), event.getRenameSchemaName(), event.getSchema());
persistService.deleteSchema(event.getDatabaseName(), event.getSchemaName());
}
diff --git a/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegistrySubscriberTest.java b/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegist [...]
index 0e3cc19a039..631a9eab31f 100644
--- a/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegistrySubscriberTest.java
+++ b/shardingsphere-mode/shardingsphere-mode-type/shardingsphere-cluster-mode/shardingsphere-cluster-mode-core/src/test/java/org/apache/shardingsphere/mode/manager/cluster/coordinator/registry/metadata/subscriber/SchemaMetaDataRegistrySubscriberTest.java
@@ -84,8 +84,8 @@ public final class SchemaMetaDataRegistrySubscriberTest {
ShardingSphereSchema schema = new ShardingSphereSchema(Collections.singletonMap("t_order", new TableMetaData()));
AlterSchemaEvent event = new AlterSchemaEvent("foo_db", "foo_schema", "new_foo_schema", schema);
schemaMetaDataRegistrySubscriber.alterSchema(event);
+ verify(persistService).persistMetaData("foo_db", "new_foo_schema", schema);
verify(persistService).deleteSchema("foo_db", "foo_schema");
- verify(persistService).persistTables("foo_db", "new_foo_schema", schema);
}
@Test
@@ -93,7 +93,7 @@ public final class SchemaMetaDataRegistrySubscriberTest {
ShardingSphereSchema schema = new ShardingSphereSchema();
AlterSchemaEvent event = new AlterSchemaEvent("foo_db", "foo_schema", "new_foo_schema", schema);
schemaMetaDataRegistrySubscriber.alterSchema(event);
+ verify(persistService).persistMetaData("foo_db", "new_foo_schema", schema);
verify(persistService).deleteSchema("foo_db", "foo_schema");
- verify(persistService).persistSchema("foo_db", "new_foo_schema");
}
}
diff --git a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/text/distsql/ral/common/updatable/RefreshTableMetadataHandler.java b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/text/distsql/ral/common/updatable/RefreshTableMetadataHandler.java
index 68bc3b30dbc..c0e41312592 100644
--- a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/text/distsql/ral/common/updatable/RefreshTableMetadataHandler.java
+++ b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/text/distsql/ral/common/updatable/RefreshTableMetadataHandler.java
@@ -54,7 +54,7 @@ public final class RefreshTableMetadataHandler extends UpdatableRALBackendHandle
contextManager.reloadMetaData(databaseName, schemaName, sqlStatement.getTableName().get());
return;
}
- contextManager.reloadMetaData(databaseName, schemaName);
+ contextManager.reloadMetaData(databaseName);
}
private String getDatabaseName() {