You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2022/04/20 12:43:33 UTC
[shardingsphere] branch master updated: Support data source config simplification (#15881) (#16957)
This is an automated email from the ASF dual-hosted git repository.
zhangliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new fa57386f74b Support data source config simplification (#15881) (#16957)
fa57386f74b is described below
commit fa57386f74b4a136f4d5d5657c1f6ce7d78b8c97
Author: galaxy <ga...@tencent.com>
AuthorDate: Wed Apr 20 20:43:20 2022 +0800
Support data source config simplification (#15881) (#16957)
* resolve code conflicts (#15881)
* remove redundant output (#15881)
---
.../shardingsphere/infra/datanode/DataNode.java | 24 +++++++++++++++----
.../pool/creator/DataSourcePoolCreator.java | 24 +++++++++++++++----
.../registry/GlobalDataSourceRegistry.java | 6 ++++-
.../loader/dialect/MySQLTableMetaDataLoader.java | 6 ++++-
.../metadata/schema/util/TableMetaDataUtil.java | 27 +++++++++++++++++++---
.../infra/datanode/DataNodeTest.java | 2 +-
.../jdbc/datasource/JDBCBackendDataSource.java | 10 ++++++++
7 files changed, 84 insertions(+), 15 deletions(-)
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java
index 67fe92cd083..4a798e3335f 100644
--- a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datanode/DataNode.java
@@ -51,16 +51,20 @@ public final class DataNode {
*/
public DataNode(final String dataNode) {
// TODO remove duplicated splitting?
- if (!isValidDataNode(dataNode)) {
+ boolean isIncludeInstance = isActualDataNodesIncludedDataSourceInstance(dataNode);
+ if (!isIncludeInstance && !isValidDataNode(dataNode, 2)) {
+ throw new ShardingSphereConfigurationException("Invalid format for actual data nodes: '%s'", dataNode);
+ }
+ if (isIncludeInstance && !isValidDataNode(dataNode, 3)) {
throw new ShardingSphereConfigurationException("Invalid format for actual data nodes: '%s'", dataNode);
}
List<String> segments = Splitter.on(DELIMITER).splitToList(dataNode);
- dataSourceName = segments.get(0);
- tableName = segments.get(1);
+ dataSourceName = isIncludeInstance ? segments.get(0) + DELIMITER + segments.get(1) : segments.get(0);
+ tableName = segments.get(isIncludeInstance ? 2 : 1);
}
- private static boolean isValidDataNode(final String dataNodeStr) {
- return dataNodeStr.contains(DELIMITER) && 2 == Splitter.on(DELIMITER).omitEmptyStrings().splitToList(dataNodeStr).size();
+ private static boolean isValidDataNode(final String dataNodeStr, final Integer tier) {
+ return dataNodeStr.contains(DELIMITER) && tier == Splitter.on(DELIMITER).omitEmptyStrings().splitToList(dataNodeStr).size();
}
@Override
@@ -99,4 +103,14 @@ public final class DataNode {
public int getFormattedTextLength() {
return dataSourceName.length() + DELIMITER.length() + tableName.length();
}
+
+ /**
+ * Is Actual data nodes three tier structure.
+ *
+ * @param actualDataNodes dataSource map
+ * @return boolean
+ */
+ public static boolean isActualDataNodesIncludedDataSourceInstance(final String actualDataNodes) {
+ return isValidDataNode(actualDataNodes, 3);
+ }
}
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/pool/creator/DataSourcePoolCreator.java b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/pool/creator/DataSourcePoolCreator.java
index ae4e944a224..21dd9739b79 100644
--- a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/pool/creator/DataSourcePoolCreator.java
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/pool/creator/DataSourcePoolCreator.java
@@ -50,7 +50,8 @@ public final class DataSourcePoolCreator {
* @return created data sources
*/
public static Map<String, DataSource> create(final Map<String, DataSourceProperties> dataSourcePropsMap) {
- return dataSourcePropsMap.entrySet().stream().collect(Collectors.toMap(Entry::getKey, entry -> create(entry.getValue()), (oldValue, currentValue) -> oldValue, LinkedHashMap::new));
+ return dataSourcePropsMap.entrySet().stream().collect(Collectors.toMap(Entry::getKey, entry -> create(entry.getKey(), entry.getValue()), (oldValue, currentValue) -> oldValue,
+ LinkedHashMap::new));
}
/**
@@ -61,8 +62,8 @@ public final class DataSourcePoolCreator {
*/
public static DataSource create(final DataSourceProperties dataSourceProps) {
if (isCanBeDataSourceAggregation(dataSourceProps)) {
- if (GlobalDataSourceRegistry.getInstance().getCachedDataSources().containsKey(dataSourceProps.getInstance())) {
- return GlobalDataSourceRegistry.getInstance().getCachedDataSources().get(dataSourceProps.getInstance());
+ if (GlobalDataSourceRegistry.getInstance().getCachedInstanceDataSources().containsKey(dataSourceProps.getInstance())) {
+ return GlobalDataSourceRegistry.getInstance().getCachedInstanceDataSources().get(dataSourceProps.getInstance());
}
}
// TODO when aggregation is enabled, some data source properties should be changed, e.g. maxPoolSize
@@ -78,7 +79,22 @@ public final class DataSourcePoolCreator {
setConfiguredFields(dataSourceProps, dataSourceReflection);
}
if (isCanBeDataSourceAggregation(dataSourceProps)) {
- GlobalDataSourceRegistry.getInstance().getCachedDataSources().put(dataSourceProps.getInstance(), result);
+ GlobalDataSourceRegistry.getInstance().getCachedInstanceDataSources().put(dataSourceProps.getInstance(), result);
+ }
+ return result;
+ }
+
+ /**
+ * Create data source.
+ *
+ * @param dataSourceName data source name
+ * @param dataSourceProps data source properties
+ * @return created data source
+ */
+ public static DataSource create(final String dataSourceName, final DataSourceProperties dataSourceProps) {
+ DataSource result = create(dataSourceProps);
+ if (!GlobalDataSourceRegistry.getInstance().getCachedDataSourceDataSources().containsKey(dataSourceName)) {
+ GlobalDataSourceRegistry.getInstance().getCachedDataSourceDataSources().put(dataSourceName, result);
}
return result;
}
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/registry/GlobalDataSourceRegistry.java b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/registry/GlobalDataSourceRegistry.java
index a60d1c957dd..c297ee4cb14 100644
--- a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/registry/GlobalDataSourceRegistry.java
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/datasource/registry/GlobalDataSourceRegistry.java
@@ -36,7 +36,11 @@ public final class GlobalDataSourceRegistry {
private static final GlobalDataSourceRegistry INSTANCE = new GlobalDataSourceRegistry();
- private volatile Map<Instance, DataSource> cachedDataSources = new LinkedHashMap<>();
+ private volatile Map<Instance, DataSource> cachedInstanceDataSources = new LinkedHashMap<>();
+
+ private volatile Map<String, DataSource> cachedDataSourceDataSources = new LinkedHashMap<>();
+
+ private volatile Map<String, String> cachedDatabaseTables = new LinkedHashMap<>();
private volatile Map<String, String> dataSourceSchema = new LinkedHashMap<>();
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/loader/dialect/MySQLTableMetaDataLoader.java b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/loader/dialect/MySQLTableMetaDataLoader.java
index c31d6af0f26..07dc1c80f8a 100644
--- a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/loader/dialect/MySQLTableMetaDataLoader.java
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/loader/dialect/MySQLTableMetaDataLoader.java
@@ -17,6 +17,7 @@
package org.apache.shardingsphere.infra.metadata.schema.loader.dialect;
+import org.apache.shardingsphere.infra.datasource.registry.GlobalDataSourceRegistry;
import org.apache.shardingsphere.infra.metadata.schema.loader.common.DataTypeLoader;
import org.apache.shardingsphere.infra.metadata.schema.loader.spi.DialectTableMetaDataLoader;
import org.apache.shardingsphere.infra.metadata.schema.model.ColumnMetaData;
@@ -101,7 +102,8 @@ public final class MySQLTableMetaDataLoader implements DialectTableMetaDataLoade
PreparedStatement preparedStatement = connection.prepareStatement(getTableMetaDataSQL(tables))) {
Map<String, Integer> dataTypes = DataTypeLoader.load(connection.getMetaData());
appendDataTypes(dataTypes);
- preparedStatement.setString(1, connection.getCatalog());
+ String databaseName = "".equals(connection.getCatalog()) ? GlobalDataSourceRegistry.getInstance().getCachedDatabaseTables().get(tables.iterator().next()) : connection.getCatalog();
+ preparedStatement.setString(1, databaseName);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
@@ -141,6 +143,8 @@ public final class MySQLTableMetaDataLoader implements DialectTableMetaDataLoade
try (Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(getIndexMetaDataSQL(tableNames))) {
preparedStatement.setString(1, connection.getCatalog());
+ String databaseName = "".equals(connection.getCatalog()) ? GlobalDataSourceRegistry.getInstance().getCachedDatabaseTables().get(tableNames.iterator().next()) : connection.getCatalog();
+ preparedStatement.setString(1, databaseName);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
String indexName = resultSet.getString("INDEX_NAME");
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/util/TableMetaDataUtil.java b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/util/TableMetaDataUtil.java
index be9b94033f2..bb9c2ed8150 100644
--- a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/util/TableMetaDataUtil.java
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/util/TableMetaDataUtil.java
@@ -19,17 +19,20 @@ package org.apache.shardingsphere.infra.metadata.schema.util;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
+import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.datanode.DataNode;
import org.apache.shardingsphere.infra.datanode.DataNodes;
+import org.apache.shardingsphere.infra.datasource.registry.GlobalDataSourceRegistry;
+import org.apache.shardingsphere.infra.exception.ShardingSphereException;
import org.apache.shardingsphere.infra.metadata.schema.builder.SchemaBuilderMaterials;
import org.apache.shardingsphere.infra.metadata.schema.loader.TableMetaDataLoaderMaterial;
import java.util.Collection;
import java.util.LinkedHashMap;
-import java.util.LinkedList;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
+import java.util.LinkedList;
/**
* Table meta data utility class.
@@ -49,17 +52,35 @@ public class TableMetaDataUtil {
Map<String, Collection<String>> dataSourceTableGroups = new LinkedHashMap<>();
DataNodes dataNodes = new DataNodes(materials.getRules());
for (String each : tableNames) {
+ checkDataSourceTypeIncludeInstanceAndSetDatabaseTableMap(materials.getDatabaseType(), dataNodes, each);
if (checkMetaDataEnable) {
addAllActualTableDataNode(materials, dataSourceTableGroups, dataNodes, each);
} else {
addOneActualTableDataNode(materials, dataSourceTableGroups, dataNodes, each);
}
}
- return dataSourceTableGroups.entrySet().stream().map(entry -> new TableMetaDataLoaderMaterial(entry.getValue(), materials.getDataSourceMap().get(entry.getKey()))).collect(Collectors.toList());
+ return dataSourceTableGroups.entrySet().stream().map(entry -> new TableMetaDataLoaderMaterial(entry.getValue(), materials.getDataSourceMap().get(entry.getKey().contains(".")
+ ? entry.getKey().split("\\.")[0]
+ : entry.getKey()))).collect(Collectors.toList());
+ }
+
+ private static void checkDataSourceTypeIncludeInstanceAndSetDatabaseTableMap(final DatabaseType databaseType, final DataNodes dataNodes, final String tableName) {
+ for (DataNode dataNode : dataNodes.getDataNodes(tableName)) {
+ if (databaseType.getName() != null && !databaseType.getName().equals("MySQL") && dataNode.getDataSourceName().contains(".")) {
+ throw new ShardingSphereException("Unsupported jdbc: '%s', actualDataNode:'%s', database type is not mysql, but actual data is three-tier structure",
+ databaseType.getJdbcUrlPrefixes(), dataNode.getDataSourceName());
+ }
+ if (dataNode.getDataSourceName().contains(".")) {
+ String database = dataNode.getDataSourceName().split("\\.")[1];
+ GlobalDataSourceRegistry.getInstance().getCachedDatabaseTables().put(dataNode.getTableName(), database);
+ }
+ }
}
private static void addOneActualTableDataNode(final SchemaBuilderMaterials materials, final Map<String, Collection<String>> dataSourceTableGroups, final DataNodes dataNodes, final String table) {
- Optional<DataNode> optional = dataNodes.getDataNodes(table).stream().filter(dataNode -> materials.getDataSourceMap().containsKey(dataNode.getDataSourceName())).findFirst();
+ Optional<DataNode> optional = dataNodes.getDataNodes(table).stream().filter(dataNode -> materials.getDataSourceMap().containsKey(dataNode.getDataSourceName().contains(".")
+ ? dataNode.getDataSourceName().split("\\.")[0]
+ : dataNode.getDataSourceName())).findFirst();
String dataSourceName = optional.map(DataNode::getDataSourceName).orElseGet(() -> materials.getDataSourceMap().keySet().iterator().next());
String tableName = optional.map(DataNode::getTableName).orElse(table);
addDataSourceTableGroups(dataSourceName, tableName, dataSourceTableGroups);
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/datanode/DataNodeTest.java b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/datanode/DataNodeTest.java
index 18ceda0ec36..b90ae726e08 100644
--- a/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/datanode/DataNodeTest.java
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/datanode/DataNodeTest.java
@@ -42,7 +42,7 @@ public final class DataNodeTest {
@Test(expected = ShardingSphereConfigurationException.class)
public void assertNewInValidDataNodeWithTwoDelimiters() {
- new DataNode("ds_0.tbl_0.tbl_1");
+ new DataNode("ds_0.db_0.tbl_0.tbl_1");
}
@Test(expected = ShardingSphereConfigurationException.class)
diff --git a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/communication/jdbc/datasource/JDBCBackendDataSource.java b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/communication/jdbc/datasource/JDBCBackendDataSource.java
index 96cc18216b2..6f7fb8031a6 100644
--- a/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/communication/jdbc/datasource/JDBCBackendDataSource.java
+++ b/shardingsphere-proxy/shardingsphere-proxy-backend/src/main/java/org/apache/shardingsphere/proxy/backend/communication/jdbc/datasource/JDBCBackendDataSource.java
@@ -70,6 +70,12 @@ public final class JDBCBackendDataSource implements BackendDataSource {
public List<Connection> getConnections(final String schemaName, final String dataSourceName,
final int connectionSize, final ConnectionMode connectionMode, final TransactionType transactionType) throws SQLException {
DataSource dataSource = ProxyContext.getInstance().getContextManager().getMetaDataContexts().getMetaData(schemaName).getResource().getDataSources().get(dataSourceName);
+ if (dataSourceName.contains(".")) {
+ String dataSourceStr = dataSourceName.split("\\.")[0];
+ if (GlobalDataSourceRegistry.getInstance().getCachedDataSourceDataSources().containsKey(dataSourceStr)) {
+ dataSource = GlobalDataSourceRegistry.getInstance().getCachedDataSourceDataSources().get(dataSourceStr);
+ }
+ }
Preconditions.checkNotNull(dataSource, "Can not get connection from datasource %s.", dataSourceName);
if (1 == connectionSize) {
return Collections.singletonList(createConnection(schemaName, dataSourceName, dataSource, transactionType));
@@ -110,6 +116,10 @@ public final class JDBCBackendDataSource implements BackendDataSource {
String databaseName = GlobalDataSourceRegistry.getInstance().getDataSourceSchema().get(schemaName + "." + dataSourceName);
result.setCatalog(databaseName);
}
+ if (dataSourceName.contains(".")) {
+ String databaseName = dataSourceName.split("\\.")[1];
+ result.setCatalog(databaseName);
+ }
return result;
}