You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2026/10/11 07:13:42 UTC
(shardingsphere) branch master updated: Fix missing table diagnostics during metadata initialization (#40153)
This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 8cce4ae7ccc Fix missing table diagnostics during metadata initialization (#40153)
8cce4ae7ccc is described below
commit 8cce4ae7cccc6c46054a15ca98b208c36149336b
Author: Liang Zhang <zh...@apache.org>
AuthorDate: Sun Oct 11 15:13:34 2026 +0800
Fix missing table diagnostics during metadata initialization (#40153)
* Fix MCP documentation link-check false positives
Replace page-relative links with language-specific root-relative paths
so existing checker rules validate published pages over HTTP while
preserving their destinations.
* Clarify PR review evidence and high-frequency invocation rules
* Fix missing table diagnostics during metadata initialization
---
.../core/metadata/data/loader/MetaDataLoader.java | 26 ----
.../metadata/data/loader/MetaDataLoaderTest.java | 123 ++----------------
.../core/src/test/resources/logback-test.xml | 2 +-
.../database/ShardingSphereDatabaseFactory.java | 2 +-
.../schema/builder/GenericSchemaBuilder.java | 89 ++++++++++++-
.../ShardingSphereDatabaseFactoryTest.java | 5 +-
.../schema/builder/GenericSchemaBuilderTest.java | 141 ++++++++++++++++++++-
7 files changed, 245 insertions(+), 143 deletions(-)
diff --git a/database/connector/core/src/main/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoader.java b/database/connector/core/src/main/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoader.java
index cafc05d3180..8106ece65a8 100644
--- a/database/connector/core/src/main/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoader.java
+++ b/database/connector/core/src/main/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoader.java
@@ -31,11 +31,9 @@ import java.sql.SQLException;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -79,7 +77,6 @@ public final class MetaDataLoader {
}
throw new SQLException(ex);
}
- checkMissingTables(materials, result);
return result;
}
@@ -118,27 +115,4 @@ public final class MetaDataLoader {
schemaMetaData.getTables().addAll(each.getTables());
}
}
-
- private static void checkMissingTables(final Collection<MetaDataLoaderMaterial> materials, final Map<String, SchemaMetaData> result) {
- Set<String> expected = new LinkedHashSet<>(materials.size(), 1F);
- for (MetaDataLoaderMaterial each : materials) {
- for (String tableName : each.getActualTableNames()) {
- expected.add(buildIdentity(each.getStorageUnitName(), each.getDefaultSchemaName(), tableName));
- }
- }
- Set<String> loaded = new LinkedHashSet<>(result.size(), 1F);
- for (SchemaMetaData each : result.values()) {
- for (TableMetaData table : each.getTables()) {
- loaded.add(buildIdentity(table.getStorageUnitName(), each.getName(), table.getName()));
- }
- }
- expected.removeAll(loaded);
- if (!expected.isEmpty()) {
- log.warn("The following tables are missing from loaded metadata: {}", expected);
- }
- }
-
- private static String buildIdentity(final String storageUnitName, final String schemaName, final String tableName) {
- return String.join(".", storageUnitName, schemaName, tableName);
- }
}
diff --git a/database/connector/core/src/test/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoaderTest.java b/database/connector/core/src/test/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoaderTest.java
index 6e0bcda1ee1..5628dd8e777 100644
--- a/database/connector/core/src/test/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoaderTest.java
+++ b/database/connector/core/src/test/java/org/apache/shardingsphere/database/connector/core/metadata/data/loader/MetaDataLoaderTest.java
@@ -28,6 +28,7 @@ import org.apache.shardingsphere.database.connector.core.metadata.data.model.Tab
import org.apache.shardingsphere.database.connector.core.type.DatabaseType;
import org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader;
import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -48,9 +49,9 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isA;
+import static org.hamcrest.Matchers.sameInstance;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -68,13 +69,14 @@ class MetaDataLoaderTest {
@SuppressWarnings({"unchecked", "rawtypes"})
@BeforeAll
static void setupLogger() {
- Logger log = (Logger) LoggerFactory.getLogger("org.apache.shardingsphere.database.connector.core.metadata.data.loader");
+ Logger log = (Logger) LoggerFactory.getLogger(MetaDataLoader.class);
ListAppender<LoggingEvent> appender = (ListAppender) log.getAppender("MetaDataLoaderTestAppender");
appenderList = appender.list;
}
@BeforeEach
- void setUp() {
+ @AfterEach
+ void clearAppenderList() {
appenderList.clear();
}
@@ -98,11 +100,18 @@ class MetaDataLoaderTest {
MetaDataLoaderMaterial material = new MetaDataLoaderMaterial(Collections.emptyList(), "dialect_sql_exception", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
DialectMetaDataLoader dialectMetaDataLoader = mock(DialectMetaDataLoader.class);
when(dialectMetaDataLoader.getType()).thenReturn(databaseType);
- when(dialectMetaDataLoader.load(any(MetaDataLoaderMaterial.class))).thenThrow(SQLException.class);
+ SQLException expectedException = new SQLException();
+ when(dialectMetaDataLoader.load(any(MetaDataLoaderMaterial.class))).thenThrow(expectedException);
try (AutoCloseable ignored = registerDialectMetaDataLoader(dialectMetaDataLoader)) {
Map<String, SchemaMetaData> actual = MetaDataLoader.load(Collections.singleton(material));
assertThat(actual.size(), is(1));
assertTrue(actual.get("foo_db").getTables().isEmpty());
+ assertThat(appenderList.size(), is(1));
+ LoggingEvent actualEvent = appenderList.get(0);
+ assertThat(actualEvent.getLevel(), is(Level.WARN));
+ assertThat(actualEvent.getFormattedMessage(), is(databaseType + " Dialect load schema meta data error, load by default."));
+ assertThat(actualEvent.getThrowableProxy(), isA(ThrowableProxy.class));
+ assertThat(((ThrowableProxy) actualEvent.getThrowableProxy()).getThrowable(), sameInstance(expectedException));
}
}
@@ -121,6 +130,7 @@ class MetaDataLoaderTest {
Map<String, SchemaMetaData> actual = MetaDataLoader.load(Collections.singleton(material));
assertThat(actual.size(), is(1));
assertTrue(actual.get("foo_db").getTables().isEmpty());
+ assertTrue(appenderList.isEmpty());
}
@Test
@@ -164,111 +174,6 @@ class MetaDataLoaderTest {
assertThat(ex.getCause().getCause(), isA(IllegalStateException.class));
}
- @Test
- void assertMissingTableWarning() throws SQLException {
- MetaDataLoaderMaterial material = new MetaDataLoaderMaterial(Collections.singleton("t_order"), "foo_ds", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- MetaDataLoader.load(Collections.singleton(material));
- assertThat(appenderList.size(), is(1));
- LoggingEvent event = appenderList.get(0);
- assertThat(event.getLevel(), is(Level.WARN));
- assertThat(event.getFormattedMessage(), is("The following tables are missing from loaded metadata: [foo_ds.foo_db.t_order]"));
- }
-
- @Test
- void assertNoMissingTableWarningWhenLoaded() throws Exception {
- MetaDataLoaderMaterial material = new MetaDataLoaderMaterial(Collections.singleton("foo_tbl"), "dialect_success", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- DialectMetaDataLoader dialectMetaDataLoader = mock(DialectMetaDataLoader.class);
- when(dialectMetaDataLoader.getType()).thenReturn(databaseType);
- when(dialectMetaDataLoader.load(material)).thenReturn(Collections.singleton(
- new SchemaMetaData("foo_db", Collections.singleton(new TableMetaData("foo_tbl", Collections.emptyList(), Collections.emptyList(), Collections.emptyList())))));
- try (AutoCloseable ignored = registerDialectMetaDataLoader(dialectMetaDataLoader)) {
- MetaDataLoader.load(Collections.singleton(material));
- assertTrue(appenderList.isEmpty());
- }
- }
-
- @Test
- void assertDialectSQLExceptionProducesWarn() throws Exception {
- MetaDataLoaderMaterial material = new MetaDataLoaderMaterial(Collections.emptyList(), "dialect_sql_exception", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- DialectMetaDataLoader dialectMetaDataLoader = mock(DialectMetaDataLoader.class);
- when(dialectMetaDataLoader.getType()).thenReturn(databaseType);
- when(dialectMetaDataLoader.load(any(MetaDataLoaderMaterial.class))).thenThrow(SQLException.class);
- try (AutoCloseable ignored = registerDialectMetaDataLoader(dialectMetaDataLoader)) {
- Map<String, SchemaMetaData> actual = MetaDataLoader.load(Collections.singleton(material));
- assertThat(actual.size(), is(1));
- assertTrue(actual.get("foo_db").getTables().isEmpty());
- assertThat(appenderList.size(), is(1));
- LoggingEvent event = appenderList.get(0);
- assertThat(event.getLevel(), is(Level.WARN));
- assertThat(event.getFormattedMessage(), containsString("Dialect load schema meta data error, load by default."));
- assertTrue(event.getThrowableProxy() instanceof ThrowableProxy);
- assertThat(((ThrowableProxy) event.getThrowableProxy()).getClassName(), is(SQLException.class.getName()));
- }
- }
-
- @Test
- void assertMultiBatchAccumulation() throws SQLException {
- MetaDataLoaderMaterial firstMaterial = new MetaDataLoaderMaterial(Collections.singleton("t_order"), "foo_ds", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- MetaDataLoaderMaterial secondMaterial = new MetaDataLoaderMaterial(Collections.singleton("t_order_item"), "foo_ds", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- MetaDataLoader.load(Arrays.asList(firstMaterial, secondMaterial));
- assertThat(appenderList.size(), is(1));
- LoggingEvent event = appenderList.get(0);
- assertThat(event.getLevel(), is(Level.WARN));
- assertThat(event.getFormattedMessage(), is("The following tables are missing from loaded metadata: [foo_ds.foo_db.t_order, foo_ds.foo_db.t_order_item]"));
- }
-
- @Test
- void assertMissingTableWarningForMultipleStorageUnits() throws SQLException {
- MetaDataLoaderMaterial firstMaterial = new MetaDataLoaderMaterial(
- Collections.singleton("t_order"), "foo_ds_1", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- MetaDataLoaderMaterial secondMaterial = new MetaDataLoaderMaterial(
- Collections.singleton("t_order"), "foo_ds_2", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- MetaDataLoader.load(Arrays.asList(firstMaterial, secondMaterial));
- assertThat(appenderList.size(), is(1));
- assertThat(appenderList.get(0).getFormattedMessage(), is("The following tables are missing from loaded metadata: [foo_ds_1.foo_db.t_order, foo_ds_2.foo_db.t_order]"));
- }
-
- @Test
- void assertMissingTableWarningWhenSameNameLoadedInOtherSchema() throws Exception {
- MetaDataLoaderMaterial material = new MetaDataLoaderMaterial(Collections.singleton("foo"), "dialect_success", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "public");
- DialectMetaDataLoader dialectMetaDataLoader = mock(DialectMetaDataLoader.class);
- when(dialectMetaDataLoader.getType()).thenReturn(databaseType);
- when(dialectMetaDataLoader.load(material)).thenReturn(Collections.singleton(
- new SchemaMetaData("archive", Collections.singleton(new TableMetaData("foo", Collections.emptyList(), Collections.emptyList(), Collections.emptyList())))));
- try (AutoCloseable ignored = registerDialectMetaDataLoader(dialectMetaDataLoader)) {
- MetaDataLoader.load(Collections.singleton(material));
- assertThat(appenderList.size(), is(1));
- assertThat(appenderList.get(0).getFormattedMessage(), is("The following tables are missing from loaded metadata: [dialect_success.public.foo]"));
- }
- }
-
- @Test
- void assertMissingTableWarningKeepsPhysicalNameCase() throws Exception {
- MetaDataLoaderMaterial material = new MetaDataLoaderMaterial(Arrays.asList("Foo", "foo"), "dialect_success", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "foo_db");
- DialectMetaDataLoader dialectMetaDataLoader = mock(DialectMetaDataLoader.class);
- when(dialectMetaDataLoader.getType()).thenReturn(databaseType);
- when(dialectMetaDataLoader.load(material)).thenReturn(Collections.singleton(
- new SchemaMetaData("foo_db", Collections.singleton(new TableMetaData("foo", Collections.emptyList(), Collections.emptyList(), Collections.emptyList())))));
- try (AutoCloseable ignored = registerDialectMetaDataLoader(dialectMetaDataLoader)) {
- MetaDataLoader.load(Collections.singleton(material));
- assertThat(appenderList.size(), is(1));
- assertThat(appenderList.get(0).getFormattedMessage(), is("The following tables are missing from loaded metadata: [dialect_success.foo_db.Foo]"));
- }
- }
-
- @Test
- void assertNoMissingTableWarningWhenSchemaAwareLoadMatches() throws Exception {
- MetaDataLoaderMaterial material = new MetaDataLoaderMaterial(Collections.singleton("foo"), "dialect_success", mock(DataSource.class, RETURNS_DEEP_STUBS), databaseType, "public");
- DialectMetaDataLoader dialectMetaDataLoader = mock(DialectMetaDataLoader.class);
- when(dialectMetaDataLoader.getType()).thenReturn(databaseType);
- when(dialectMetaDataLoader.load(material)).thenReturn(Collections.singleton(
- new SchemaMetaData("public", Collections.singleton(new TableMetaData("foo", Collections.emptyList(), Collections.emptyList(), Collections.emptyList())))));
- try (AutoCloseable ignored = registerDialectMetaDataLoader(dialectMetaDataLoader)) {
- MetaDataLoader.load(Collections.singleton(material));
- assertTrue(appenderList.isEmpty());
- }
- }
-
@SneakyThrows(ReflectiveOperationException.class)
private AutoCloseable registerDialectMetaDataLoader(final DialectMetaDataLoader service) {
Map<Class<?>, Object> registeredServices = getRegisteredServices();
diff --git a/database/connector/core/src/test/resources/logback-test.xml b/database/connector/core/src/test/resources/logback-test.xml
index b08cdc0a436..4546eff7bbe 100644
--- a/database/connector/core/src/test/resources/logback-test.xml
+++ b/database/connector/core/src/test/resources/logback-test.xml
@@ -28,7 +28,7 @@
</logger>
<appender name="MetaDataLoaderTestAppender" class="ch.qos.logback.core.read.ListAppender" />
- <logger name="org.apache.shardingsphere.database.connector.core.metadata.data.loader" level="warn" additivity="false">
+ <logger name="org.apache.shardingsphere.database.connector.core.metadata.data.loader.MetaDataLoader" level="warn" additivity="false">
<appender-ref ref="MetaDataLoaderTestAppender" />
</logger>
diff --git a/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactory.java b/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactory.java
index 0e0a7a9d2b1..7121bc503ce 100644
--- a/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactory.java
+++ b/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactory.java
@@ -137,7 +137,7 @@ public final class ShardingSphereDatabaseFactory {
ResourceMetaData resourceMetaData = new ResourceMetaData(databaseConfig.getDataSources(), databaseConfig.getStorageUnits());
DatabaseIdentifierContext identifierContext = DatabaseIdentifierContextFactory.create(protocolType, resourceMetaData, props);
Collection<ShardingSphereRule> databaseRules = DatabaseRulesBuilder.build(name, protocolType, databaseConfig, computeNodeInstanceContext, resourceMetaData);
- Map<String, ShardingSphereSchema> schemas = new ConcurrentHashMap<>(GenericSchemaBuilder.build(protocolType,
+ Map<String, ShardingSphereSchema> schemas = new ConcurrentHashMap<>(GenericSchemaBuilder.buildWithInventoryCheck(protocolType,
new GenericSchemaBuilderMaterial(resourceMetaData.getStorageUnits(), databaseRules, props, DefaultSchemaNameResolver.resolveProtocol(protocolType, name),
identifierContext, revisionCandidateSchemas)));
SystemSchemaBuilder.build(name, protocolType, props).forEach(schemas::putIfAbsent);
diff --git a/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilder.java b/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilder.java
index 31a1541a0a5..3e2f98cd2b4 100644
--- a/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilder.java
+++ b/infra/common/src/main/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilder.java
@@ -19,13 +19,18 @@ package org.apache.shardingsphere.infra.metadata.database.schema.builder;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import org.apache.shardingsphere.database.connector.core.metadata.data.loader.MetaDataLoader;
import org.apache.shardingsphere.database.connector.core.metadata.data.loader.MetaDataLoaderMaterial;
import org.apache.shardingsphere.database.connector.core.metadata.data.model.SchemaMetaData;
import org.apache.shardingsphere.database.connector.core.metadata.data.model.TableMetaData;
import org.apache.shardingsphere.database.connector.core.metadata.database.metadata.DialectDatabaseMetaData;
+import org.apache.shardingsphere.database.connector.core.metadata.database.metadata.option.schema.DialectSchemaOption;
import org.apache.shardingsphere.database.connector.core.spi.DatabaseTypedSPILoader;
import org.apache.shardingsphere.database.connector.core.type.DatabaseType;
+import org.apache.shardingsphere.database.connector.core.type.DatabaseTypeRegistry;
+import org.apache.shardingsphere.infra.datanode.DataNode;
+import org.apache.shardingsphere.infra.datanode.DataNodes;
import org.apache.shardingsphere.infra.metadata.database.resource.unit.StorageUnit;
import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereSchema;
import org.apache.shardingsphere.infra.metadata.database.schema.reviser.MetaDataReviseEngine;
@@ -37,15 +42,19 @@ import java.sql.SQLException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
/**
* Generic schema builder.
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
+@Slf4j
public final class GenericSchemaBuilder {
/**
@@ -70,10 +79,18 @@ public final class GenericSchemaBuilder {
* @throws SQLException SQL exception
*/
public static Map<String, ShardingSphereSchema> build(final Collection<String> tableNames, final DatabaseType protocolType, final GenericSchemaBuilderMaterial material) throws SQLException {
+ return build(tableNames, protocolType, material, false);
+ }
+
+ private static Map<String, ShardingSphereSchema> build(final Collection<String> tableNames, final DatabaseType protocolType, final GenericSchemaBuilderMaterial material,
+ final boolean checkInventory) throws SQLException {
Collection<MetaDataLoaderMaterial> loaderMaterials = SchemaMetaDataUtils.getMetaDataLoaderMaterials(tableNames, protocolType, material);
Map<String, SchemaMetaData> result = loadSchemas(loaderMaterials);
+ if (checkInventory) {
+ checkMissingTables(tableNames, loaderMaterials, result, material.getRules());
+ }
if (!isSchemaCompatible(protocolType, material.getStorageUnits())) {
- result = translate(result, protocolType, material, loaderMaterials);
+ result = translate(result, material, loaderMaterials);
}
return revise(result, material, protocolType);
}
@@ -90,6 +107,61 @@ public final class GenericSchemaBuilder {
return materials.isEmpty() ? Collections.emptyMap() : MetaDataLoader.load(materials);
}
+ private static void checkMissingTables(final Collection<String> tableNames, final Collection<MetaDataLoaderMaterial> materials, final Map<String, SchemaMetaData> schemaMetaDataMap,
+ final Collection<ShardingSphereRule> rules) {
+ Map<String, Set<String>> requestedTables = getRequestedTables(materials);
+ Map<String, Optional<String>> storageSchemaNames = getStorageSchemaNames(materials);
+ Set<DataNode> loadedTables = schemaMetaDataMap.values().stream()
+ .flatMap(each -> each.getTables().stream().map(table -> new DataNode(table.getStorageUnitName(), each.getName(), table.getName()))).collect(Collectors.toSet());
+ DataNodes dataNodes = new DataNodes(rules);
+ Set<String> missingTables = new LinkedHashSet<>();
+ for (String each : tableNames) {
+ if (dataNodes.isDataNodeTableNameLoadedFromStorage(each)) {
+ addMissingTables(dataNodes.getDataNodes(each), requestedTables, storageSchemaNames, loadedTables, missingTables);
+ }
+ }
+ if (!missingTables.isEmpty()) {
+ log.warn("The following tables are missing from loaded metadata: {}", missingTables);
+ }
+ }
+
+ private static Map<String, Set<String>> getRequestedTables(final Collection<MetaDataLoaderMaterial> materials) {
+ Map<String, Set<String>> result = new LinkedHashMap<>();
+ for (MetaDataLoaderMaterial each : materials) {
+ result.computeIfAbsent(each.getStorageUnitName(), key -> new HashSet<>(each.getActualTableNames().size(), 1F)).addAll(each.getActualTableNames());
+ }
+ return result;
+ }
+
+ private static Map<String, Optional<String>> getStorageSchemaNames(final Collection<MetaDataLoaderMaterial> materials) {
+ Map<String, Optional<String>> result = new LinkedHashMap<>();
+ Set<String> storageUnitNames = new HashSet<>();
+ for (MetaDataLoaderMaterial each : materials) {
+ if (storageUnitNames.add(each.getStorageUnitName())) {
+ DialectSchemaOption schemaOption = new DatabaseTypeRegistry(each.getStorageType()).getDialectDatabaseMetaData().getSchemaOption();
+ if (schemaOption.isSchemaAvailable()) {
+ result.put(each.getStorageUnitName(), Optional.empty());
+ } else if (!schemaOption.getDefaultSchema().isPresent()) {
+ result.put(each.getStorageUnitName(), Optional.of(each.getDefaultSchemaName()));
+ }
+ }
+ }
+ return result;
+ }
+
+ private static void addMissingTables(final Collection<DataNode> inventoriedDataNodes, final Map<String, Set<String>> requestedTables, final Map<String, Optional<String>> storageSchemaNames,
+ final Set<DataNode> loadedTables, final Set<String> missingTables) {
+ for (DataNode each : inventoriedDataNodes) {
+ Set<String> requestedTableNames = requestedTables.get(each.getDataSourceName());
+ if (null != requestedTableNames && requestedTableNames.contains(each.getTableName()) && storageSchemaNames.containsKey(each.getDataSourceName())) {
+ String schemaName = storageSchemaNames.get(each.getDataSourceName()).orElse(each.getSchemaName());
+ if (!loadedTables.contains(new DataNode(each.getDataSourceName(), schemaName, each.getTableName()))) {
+ missingTables.add(String.join(".", each.getDataSourceName(), each.getSchemaName(), each.getTableName()));
+ }
+ }
+ }
+ }
+
private static boolean isSchemaCompatible(final DatabaseType protocolType, final Map<String, StorageUnit> storageUnits) {
return isSameProtocolAndStorageTypes(protocolType, storageUnits) || isSchemaAvailable(protocolType)
&& storageUnits.values().stream().map(StorageUnit::getStorageType).allMatch(GenericSchemaBuilder::isSchemaAvailable);
@@ -104,7 +176,7 @@ public final class GenericSchemaBuilder {
.map(each -> each.getSchemaOption().isSchemaAvailable()).orElse(false);
}
- private static Map<String, SchemaMetaData> translate(final Map<String, SchemaMetaData> schemaMetaDataMap, final DatabaseType protocolType, final GenericSchemaBuilderMaterial material,
+ private static Map<String, SchemaMetaData> translate(final Map<String, SchemaMetaData> schemaMetaDataMap, final GenericSchemaBuilderMaterial material,
final Collection<MetaDataLoaderMaterial> loaderMaterials) {
Collection<TableMetaData> tableMetaDataList = new LinkedList<>();
for (String each : loaderMaterials.stream().map(MetaDataLoaderMaterial::getDefaultSchemaName).distinct().collect(Collectors.toList())) {
@@ -119,4 +191,17 @@ public final class GenericSchemaBuilder {
.filter(each -> each.getAttributes().findAttribute(TableMapperRuleAttribute.class).isPresent()).collect(Collectors.toList());
return new MetaDataReviseEngine(rules, protocolType).revise(schemaMetaDataMap, material);
}
+
+ /**
+ * Build generic schemas and warn about missing inventoried single tables.
+ * Call this after constructing fresh database rules, before mutable data nodes contain SQL refresh candidates.
+ *
+ * @param protocolType database type
+ * @param material generic schema builder material with freshly inventoried data nodes
+ * @return generic schema map
+ * @throws SQLException SQL exception
+ */
+ public static Map<String, ShardingSphereSchema> buildWithInventoryCheck(final DatabaseType protocolType, final GenericSchemaBuilderMaterial material) throws SQLException {
+ return build(getAllTableNames(material.getRules()), protocolType, material, true);
+ }
}
diff --git a/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactoryTest.java b/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactoryTest.java
index 553c8482118..9705d9f3d1c 100644
--- a/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactoryTest.java
+++ b/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/ShardingSphereDatabaseFactoryTest.java
@@ -90,7 +90,7 @@ class ShardingSphereDatabaseFactoryTest {
mockedDefaultSchemaNameResolver.when(() -> DefaultSchemaNameResolver.resolveProtocol(protocolType, "foo_db")).thenReturn("foo_schema");
mockedRulesBuilder.when(() -> DatabaseRulesBuilder.build(eq("foo_db"), eq(protocolType), eq(databaseConfig), eq(computeNodeInstanceContext), any(ResourceMetaData.class)))
.thenReturn(Collections.singleton(mock(ShardingSphereRule.class)));
- mockedGenericSchemaBuilder.when(() -> GenericSchemaBuilder.build(eq(protocolType), any(GenericSchemaBuilderMaterial.class)))
+ mockedGenericSchemaBuilder.when(() -> GenericSchemaBuilder.buildWithInventoryCheck(eq(protocolType), any(GenericSchemaBuilderMaterial.class)))
.thenReturn(Collections.singletonMap("foo_schema", new ShardingSphereSchema("foo_schema", protocolType)));
mockedSystemSchemaBuilder.when(() -> SystemSchemaBuilder.build("foo_db", protocolType, props)).thenReturn(systemSchemas);
ShardingSphereDatabase actual = ShardingSphereDatabaseFactory.create("foo_db", protocolType, databaseConfig, props, computeNodeInstanceContext);
@@ -118,8 +118,7 @@ class ShardingSphereDatabaseFactoryTest {
mockedDefaultSchemaNameResolver.when(() -> DefaultSchemaNameResolver.resolveProtocol(protocolType, "foo_db")).thenReturn("foo_schema");
mockedRulesBuilder.when(() -> DatabaseRulesBuilder.build(eq("foo_db"), eq(protocolType), eq(databaseConfig), eq(computeNodeInstanceContext), any(ResourceMetaData.class)))
.thenReturn(Collections.singleton(mock(ShardingSphereRule.class)));
- mockedGenericSchemaBuilder.when(() -> GenericSchemaBuilder.build(eq(protocolType),
- argThat(material -> material.getRevisionCandidateSchemas().contains(candidateSchema))))
+ mockedGenericSchemaBuilder.when(() -> GenericSchemaBuilder.buildWithInventoryCheck(eq(protocolType), argThat(material -> material.getRevisionCandidateSchemas().contains(candidateSchema))))
.thenReturn(Collections.singletonMap("foo_schema", new ShardingSphereSchema("foo_schema", protocolType)));
mockedSystemSchemaBuilder.when(() -> SystemSchemaBuilder.build("foo_db", protocolType, props)).thenReturn(Collections.emptyMap());
ShardingSphereDatabase actual = ShardingSphereDatabaseFactory.createWithRevisionCandidateSchemas(
diff --git a/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilderTest.java b/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilderTest.java
index b931745a7a3..327962bd1ba 100644
--- a/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilderTest.java
+++ b/infra/common/src/test/java/org/apache/shardingsphere/infra/metadata/database/schema/builder/GenericSchemaBuilderTest.java
@@ -17,20 +17,30 @@
package org.apache.shardingsphere.infra.metadata.database.schema.builder;
+import ch.qos.logback.classic.Level;
import org.apache.shardingsphere.database.connector.core.metadata.data.loader.MetaDataLoader;
import org.apache.shardingsphere.database.connector.core.metadata.data.loader.MetaDataLoaderMaterial;
import org.apache.shardingsphere.database.connector.core.metadata.data.model.SchemaMetaData;
import org.apache.shardingsphere.database.connector.core.metadata.data.model.TableMetaData;
import org.apache.shardingsphere.database.connector.core.type.DatabaseType;
import org.apache.shardingsphere.infra.config.props.ConfigurationProperties;
+import org.apache.shardingsphere.infra.config.props.ConfigurationPropertyKey;
+import org.apache.shardingsphere.infra.datanode.DataNode;
+import org.apache.shardingsphere.infra.metadata.database.resource.ResourceMetaData;
import org.apache.shardingsphere.infra.metadata.database.resource.unit.StorageUnit;
import org.apache.shardingsphere.infra.metadata.database.schema.model.ShardingSphereSchema;
import org.apache.shardingsphere.infra.metadata.identifier.DatabaseIdentifierContextFactory;
import org.apache.shardingsphere.infra.rule.ShardingSphereRule;
import org.apache.shardingsphere.infra.rule.attribute.RuleAttributes;
+import org.apache.shardingsphere.infra.rule.attribute.datanode.DataNodeRuleAttribute;
import org.apache.shardingsphere.infra.rule.attribute.table.TableMapperRuleAttribute;
import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader;
+import org.apache.shardingsphere.infra.util.props.PropertiesBuilder;
+import org.apache.shardingsphere.infra.util.props.PropertiesBuilder.Property;
import org.apache.shardingsphere.test.infra.fixture.jdbc.MockedDataSource;
+import org.apache.shardingsphere.test.infra.framework.extension.log.LogCaptureAssertion;
+import org.apache.shardingsphere.test.infra.framework.extension.log.LogCaptureExtension;
+import org.apache.shardingsphere.test.infra.framework.extension.log.LogCaptureSettings;
import org.apache.shardingsphere.test.infra.framework.extension.mock.AutoMockExtension;
import org.apache.shardingsphere.test.infra.framework.extension.mock.StaticMockSettings;
import org.junit.jupiter.api.BeforeEach;
@@ -47,6 +57,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -61,10 +72,12 @@ import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-@ExtendWith(AutoMockExtension.class)
+@ExtendWith({AutoMockExtension.class, LogCaptureExtension.class})
+@LogCaptureSettings("org.apache.shardingsphere.infra.metadata.database.schema.builder.GenericSchemaBuilder")
@MockitoSettings(strictness = Strictness.LENIENT)
@StaticMockSettings(MetaDataLoader.class)
class GenericSchemaBuilderTest {
@@ -79,6 +92,8 @@ class GenericSchemaBuilderTest {
private static final DatabaseType ORACLE_DATABASE_TYPE = TypedSPILoader.getService(DatabaseType.class, "Oracle");
+ private static final DatabaseType SQL_SERVER_DATABASE_TYPE = TypedSPILoader.getService(DatabaseType.class, "SQLServer");
+
private final DatabaseType databaseType = TypedSPILoader.getService(DatabaseType.class, "FIXTURE");
private GenericSchemaBuilderMaterial material;
@@ -251,4 +266,128 @@ class GenericSchemaBuilderTest {
assertTrue(actual.getTable("foo_tbl").getAllColumns().isEmpty());
assertTrue(actual.getTable("bar_tbl").getAllColumns().isEmpty());
}
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("inventoryCheckArgs")
+ void assertBuildWithInventoryCheck(final String name, final DatabaseType storageType, final String defaultSchemaName, final Collection<DataNode> inventory, final boolean checkAll,
+ final Map<String, SchemaMetaData> loadedSchemas, final Collection<String> expectedMissing, final LogCaptureAssertion logCaptureAssertion) throws SQLException {
+ GenericSchemaBuilderMaterial inventoryMaterial = createInventoryMaterial(storageType, defaultSchemaName, inventory, checkAll);
+ when(MetaDataLoader.load(any())).thenReturn(loadedSchemas);
+ GenericSchemaBuilder.buildWithInventoryCheck(storageType, inventoryMaterial);
+ logCaptureAssertion.assertLogCount(expectedMissing.isEmpty() ? 0 : 1);
+ if (!expectedMissing.isEmpty()) {
+ logCaptureAssertion.assertLogContent(0, Level.WARN, "The following tables are missing from loaded metadata: " + expectedMissing, true);
+ }
+ }
+
+ private GenericSchemaBuilderMaterial createInventoryMaterial(final DatabaseType storageType, final String defaultSchemaName, final Collection<DataNode> inventory,
+ final boolean checkAll) throws SQLException {
+ DataNodeRuleAttribute dataNodeRuleAttribute = mock(DataNodeRuleAttribute.class);
+ when(dataNodeRuleAttribute.getDataNodesByTableName(anyString())).thenAnswer(invocation -> {
+ String tableName = invocation.getArgument(0);
+ return inventory.stream().filter(each -> each.getTableName().equals(tableName)).collect(Collectors.toList());
+ });
+ when(dataNodeRuleAttribute.isDataNodeTableNameLoadedFromStorage()).thenReturn(true);
+ TableMapperRuleAttribute tableMapperRuleAttribute = mock(TableMapperRuleAttribute.class);
+ when(tableMapperRuleAttribute.getLogicTableNames()).thenReturn(inventory.stream().map(DataNode::getTableName).distinct().collect(Collectors.toList()));
+ ShardingSphereRule rule = mock(ShardingSphereRule.class);
+ when(rule.getAttributes()).thenReturn(new RuleAttributes(dataNodeRuleAttribute, tableMapperRuleAttribute));
+ Map<String, StorageUnit> storageUnits = new LinkedHashMap<>();
+ for (DataNode each : inventory) {
+ if (!storageUnits.containsKey(each.getDataSourceName())) {
+ StorageUnit storageUnit = mock(StorageUnit.class);
+ DataSource dataSource = MYSQL_DATABASE_TYPE.equals(storageType) ? mockMySQLDataSource(0) : new MockedDataSource();
+ when(storageUnit.getStorageType()).thenReturn(storageType);
+ when(storageUnit.getDataSource()).thenReturn(dataSource);
+ storageUnits.put(each.getDataSourceName(), storageUnit);
+ }
+ }
+ ConfigurationProperties props = new ConfigurationProperties(PropertiesBuilder.build(
+ new Property(ConfigurationPropertyKey.CHECK_TABLE_METADATA_ENABLED.getKey(), Boolean.toString(checkAll)),
+ new Property(ConfigurationPropertyKey.LOAD_TABLE_METADATA_BATCH_SIZE.getKey(), "1")));
+ return new GenericSchemaBuilderMaterial(storageUnits, Collections.singleton(rule), props, defaultSchemaName, DatabaseIdentifierContextFactory.createDefault());
+ }
+
+ private static Stream<Arguments> inventoryCheckArgs() {
+ DataNode mysqlTable = new DataNode("foo_ds", "foo_db", "foo_tbl");
+ DataNode publicTable = new DataNode("foo_ds", "public", "foo_tbl");
+ DataNode archiveTable = new DataNode("foo_ds", "archive", "foo_tbl");
+ DataNode h2Table = new DataNode("foo_ds", "FOO_DB", "FOO_TBL");
+ return Stream.of(
+ Arguments.of("missing MySQL single table", MYSQL_DATABASE_TYPE, "foo_db", Collections.singleton(mysqlTable), false, Collections.emptyMap(),
+ Collections.singleton("foo_ds.foo_db.foo_tbl")),
+ Arguments.of("loaded MySQL single table", MYSQL_DATABASE_TYPE, "foo_db", Collections.singleton(mysqlTable), false,
+ Collections.singletonMap("foo_db", createInventorySchema("foo_db", mysqlTable)), Collections.emptyList()),
+ Arguments.of("all requested batches loaded", MYSQL_DATABASE_TYPE, "foo_db", Arrays.asList(mysqlTable, new DataNode("foo_ds", "foo_db", "bar_tbl")), true,
+ Collections.singletonMap("foo_db", createInventorySchema("foo_db", mysqlTable, new DataNode("foo_ds", "foo_db", "bar_tbl"))), Collections.emptyList()),
+ Arguments.of("same table missing in another storage unit", MYSQL_DATABASE_TYPE, "foo_db", Arrays.asList(mysqlTable, new DataNode("bar_ds", "foo_db", "foo_tbl")), true,
+ Collections.singletonMap("foo_db", createInventorySchema("foo_db", mysqlTable)), Collections.singleton("bar_ds.foo_db.foo_tbl")),
+ Arguments.of("unrequested storage unit excluded", MYSQL_DATABASE_TYPE, "foo_db", Arrays.asList(mysqlTable, new DataNode("bar_ds", "foo_db", "foo_tbl")), false,
+ Collections.singletonMap("foo_db", createInventorySchema("foo_db", mysqlTable)), Collections.emptyList()),
+ Arguments.of("nondefault schema loaded with representative selection", POSTGRESQL_DATABASE_TYPE, "public", Collections.singleton(archiveTable), false,
+ Collections.singletonMap("archive", createInventorySchema("archive", archiveTable)), Collections.emptyList()),
+ Arguments.of("nondefault schema loaded with all-node selection", POSTGRESQL_DATABASE_TYPE, "public", Collections.singleton(archiveTable), true,
+ Collections.singletonMap("archive", createInventorySchema("archive", archiveTable)), Collections.emptyList()),
+ Arguments.of("missing nondefault schema", POSTGRESQL_DATABASE_TYPE, "public", Arrays.asList(publicTable, archiveTable), true,
+ Collections.singletonMap("public", createInventorySchema("public", publicTable)), Collections.singleton("foo_ds.archive.foo_tbl")),
+ Arguments.of("missing nondefault schema with representative selection", POSTGRESQL_DATABASE_TYPE, "public", Arrays.asList(publicTable, archiveTable), false,
+ Collections.singletonMap("public", createInventorySchema("public", publicTable)), Collections.singleton("foo_ds.archive.foo_tbl")),
+ Arguments.of("case-distinct physical tables", POSTGRESQL_DATABASE_TYPE, "public", Arrays.asList(publicTable, new DataNode("foo_ds", "public", "FOO_TBL")), true,
+ Collections.singletonMap("public", createInventorySchema("public", publicTable)), Collections.singleton("foo_ds.public.FOO_TBL")),
+ Arguments.of("dotted names retain separate identities", POSTGRESQL_DATABASE_TYPE, "public",
+ Arrays.asList(new DataNode("foo_ds", "public.bar", "foo_tbl"), new DataNode("foo_ds", "public", "bar.foo_tbl")), true,
+ Collections.singletonMap("public", createInventorySchema("public", new DataNode("foo_ds", "public", "bar.foo_tbl"))), Collections.singleton("foo_ds.public.bar.foo_tbl")),
+ Arguments.of("H2 schema alias mapped to loader schema", H2_DATABASE_TYPE, "foo_db", Collections.singleton(h2Table), false,
+ Collections.singletonMap("foo_db", createInventorySchema("foo_db", h2Table)), Collections.emptyList()),
+ Arguments.of("missing H2 inventoried table", H2_DATABASE_TYPE, "foo_db", Collections.singleton(h2Table), false, Collections.emptyMap(), Collections.singleton("foo_ds.FOO_DB.FOO_TBL")),
+ Arguments.of("Oracle schema alias mapped to loader schema", ORACLE_DATABASE_TYPE, "foo_db", Collections.singleton(h2Table), false,
+ Collections.singletonMap("foo_db", createInventorySchema("foo_db", h2Table)), Collections.emptyList()),
+ Arguments.of("unproven native schema excluded", SQL_SERVER_DATABASE_TYPE, "dbo", Collections.singleton(archiveTable), true, Collections.emptyMap(), Collections.emptyList()),
+ Arguments.of("empty inventory", POSTGRESQL_DATABASE_TYPE, "public", Collections.emptyList(), false, Collections.emptyMap(), Collections.emptyList()));
+ }
+
+ private static SchemaMetaData createInventorySchema(final String schemaName, final DataNode... dataNodes) {
+ Collection<TableMetaData> tables = new ArrayList<>(dataNodes.length);
+ for (DataNode each : dataNodes) {
+ TableMetaData tableMetaData = new TableMetaData(each.getTableName(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
+ tableMetaData.setStorageUnitName(each.getDataSourceName());
+ tables.add(tableMetaData);
+ }
+ return new SchemaMetaData(schemaName, tables);
+ }
+
+ @Test
+ void assertBuildWithInventoryCheckForConfiguredTables(final LogCaptureAssertion logCaptureAssertion) throws SQLException {
+ GenericSchemaBuilderMaterial inventoryMaterial = createInventoryMaterial(H2_DATABASE_TYPE, "foo_db", Collections.singleton(new DataNode("foo_ds", "foo_db", "foo_tbl")), false);
+ when(inventoryMaterial.getRules().iterator().next().getAttributes().getAttribute(DataNodeRuleAttribute.class).isDataNodeTableNameLoadedFromStorage()).thenReturn(false);
+ when(MetaDataLoader.load(any())).thenReturn(Collections.singletonMap("foo_db", createInventorySchema("foo_db", new DataNode("foo_ds", "foo_db", "FOO_TBL"))));
+ GenericSchemaBuilder.buildWithInventoryCheck(H2_DATABASE_TYPE, inventoryMaterial);
+ logCaptureAssertion.assertLogCount(0);
+ }
+
+ @Test
+ void assertBuildWithInventoryCheckForUnmatchedFallback(final LogCaptureAssertion logCaptureAssertion) throws SQLException {
+ GenericSchemaBuilderMaterial inventoryMaterial = createInventoryMaterial(POSTGRESQL_DATABASE_TYPE, "public", Collections.singleton(new DataNode("foo_ds", "archive", "Foo_Tbl")), false);
+ GenericSchemaBuilderMaterial actualMaterial = new GenericSchemaBuilderMaterial(inventoryMaterial.getStorageUnits(), inventoryMaterial.getRules(), inventoryMaterial.getProps(), "public",
+ DatabaseIdentifierContextFactory.create(MYSQL_DATABASE_TYPE, new ResourceMetaData(Collections.emptyMap(), inventoryMaterial.getStorageUnits()), inventoryMaterial.getProps()));
+ when(MetaDataLoader.load(any())).thenReturn(Collections.emptyMap());
+ GenericSchemaBuilder.buildWithInventoryCheck(MYSQL_DATABASE_TYPE, actualMaterial);
+ logCaptureAssertion.assertLogCount(0);
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("refreshCandidateArgs")
+ void assertBuildWithRefreshCandidate(final String name, final boolean fullLoad, final LogCaptureAssertion logCaptureAssertion) throws SQLException {
+ GenericSchemaBuilderMaterial inventoryMaterial = createInventoryMaterial(H2_DATABASE_TYPE, "foo_db", Collections.singleton(new DataNode("foo_ds", "foo_db", "foo_tbl")), false);
+ when(MetaDataLoader.load(any())).thenReturn(Collections.singletonMap("foo_db", createInventorySchema("foo_db", new DataNode("foo_ds", "foo_db", "FOO_TBL"))));
+ Map<String, ShardingSphereSchema> actual = fullLoad
+ ? GenericSchemaBuilder.build(H2_DATABASE_TYPE, inventoryMaterial)
+ : GenericSchemaBuilder.build(Collections.singleton("foo_tbl"), H2_DATABASE_TYPE, inventoryMaterial);
+ assertSchemaTable(actual, "foo_db", "FOO_TBL");
+ logCaptureAssertion.assertLogCount(0);
+ }
+
+ private static Stream<Arguments> refreshCandidateArgs() {
+ return Stream.of(Arguments.of("full rebuild with SQL candidate", true), Arguments.of("incremental DDL refresh", false));
+ }
}