You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2021/03/25 03:49:28 UTC
[shardingsphere] branch master updated: Add
PostgreSQLTableMetaDataLoader (#9808)
This is an automated email from the ASF dual-hosted git repository.
zhangliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 19f8859 Add PostgreSQLTableMetaDataLoader (#9808)
19f8859 is described below
commit 19f885942c868eb235fad874112cbed9019c4023
Author: sandynz <42...@users.noreply.github.com>
AuthorDate: Thu Mar 25 11:48:50 2021 +0800
Add PostgreSQLTableMetaDataLoader (#9808)
* Add PostgreSQLTableMetaDataLoader
* Add test case for PostgreSQLTableMetaDataLoader
---
.../dialect/PostgreSQLTableMetaDataLoader.java | 147 +++++++++++++++++++++
...a.schema.builder.spi.DialectTableMetaDataLoader | 1 +
.../dialect/PostgreSQLTableMetaDataLoaderTest.java | 141 ++++++++++++++++++++
3 files changed, 289 insertions(+)
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/builder/loader/dialect/PostgreSQLTableMetaDataLoader.java b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/builder/loader/dialect/PostgreSQLTableMetaDataLoader.java
new file mode 100644
index 0000000..3b2af1c
--- /dev/null
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/main/java/org/apache/shardingsphere/infra/metadata/schema/builder/loader/dialect/PostgreSQLTableMetaDataLoader.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.infra.metadata.schema.builder.loader.dialect;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.sql.DataSource;
+import org.apache.shardingsphere.infra.metadata.schema.builder.loader.DataTypeLoader;
+import org.apache.shardingsphere.infra.metadata.schema.builder.spi.DialectTableMetaDataLoader;
+import org.apache.shardingsphere.infra.metadata.schema.model.ColumnMetaData;
+import org.apache.shardingsphere.infra.metadata.schema.model.IndexMetaData;
+import org.apache.shardingsphere.infra.metadata.schema.model.TableMetaData;
+
+/**
+ * Table meta data loader for PostgreSQL.
+ */
+public final class PostgreSQLTableMetaDataLoader implements DialectTableMetaDataLoader {
+
+ private static final String BASIC_TABLE_META_DATA_SQL = "SELECT table_name, column_name, data_type, udt_name, column_default FROM information_schema.columns WHERE table_schema = ?";
+
+ private static final String TABLE_META_DATA_SQL_WITH_EXISTED_TABLES = BASIC_TABLE_META_DATA_SQL + " AND table_name NOT IN (%s)";
+
+ private static final String PRIMARY_KEY_META_DATA_SQL = "SELECT tc.table_name, kc.column_name FROM information_schema.table_constraints tc"
+ + " JOIN information_schema.key_column_usage kc"
+ + " ON kc.table_schema = tc.table_schema AND kc.table_name = tc.table_name AND kc.constraint_name = tc.constraint_name"
+ + " WHERE tc.constraint_type = 'PRIMARY KEY' AND kc.ordinal_position IS NOT NULL AND kc.table_schema = ?";
+
+ private static final String BASIC_INDEX_META_DATA_SQL = "SELECT tablename, indexname FROM pg_indexes WHERE schemaname = ?";
+
+ @Override
+ public Map<String, TableMetaData> load(final DataSource dataSource, final Collection<String> existedTables) throws SQLException {
+ return loadTableMetaDataMap(dataSource, existedTables);
+ }
+
+ private Map<String, TableMetaData> loadTableMetaDataMap(final DataSource dataSource, final Collection<String> existedTables) throws SQLException {
+ Map<String, TableMetaData> result = new LinkedHashMap<>();
+ Map<String, Collection<IndexMetaData>> indexMetaDataMap = loadIndexMetaDataMap(dataSource);
+ for (Entry<String, Collection<ColumnMetaData>> entry : loadColumnMetaDataMap(dataSource, existedTables).entrySet()) {
+ Collection<IndexMetaData> indexMetaDataList = indexMetaDataMap.get(entry.getKey());
+ if (null == indexMetaDataList) {
+ indexMetaDataList = Collections.emptyList();
+ }
+ result.put(entry.getKey(), new TableMetaData(entry.getValue(), indexMetaDataList));
+ }
+ return result;
+ }
+
+ private Map<String, Collection<ColumnMetaData>> loadColumnMetaDataMap(final DataSource dataSource, final Collection<String> existedTables) throws SQLException {
+ Map<String, Collection<ColumnMetaData>> result = new HashMap<>();
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement preparedStatement = connection.prepareStatement(getTableMetaDataSQL(existedTables))) {
+ Map<String, Integer> dataTypes = DataTypeLoader.load(connection.getMetaData());
+ Set<String> primaryKeys = loadPrimaryKeys(connection);
+ preparedStatement.setString(1, connection.getSchema());
+ try (ResultSet resultSet = preparedStatement.executeQuery()) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("table_name");
+ Collection<ColumnMetaData> columns = result.computeIfAbsent(tableName, k -> new LinkedList<>());
+ ColumnMetaData columnMetaData = loadColumnMetaData(dataTypes, primaryKeys, resultSet);
+ columns.add(columnMetaData);
+ }
+ }
+ }
+ return result;
+ }
+
+ private Set<String> loadPrimaryKeys(final Connection connection) throws SQLException {
+ Set<String> primaryKeys = new HashSet<>();
+ try (PreparedStatement preparedStatement = connection.prepareStatement(PRIMARY_KEY_META_DATA_SQL)) {
+ preparedStatement.setString(1, connection.getSchema());
+ try (ResultSet resultSet = preparedStatement.executeQuery()) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("table_name");
+ String columnName = resultSet.getString("column_name");
+ primaryKeys.add(tableName + "," + columnName);
+ }
+ }
+ }
+ return primaryKeys;
+ }
+
+ private ColumnMetaData loadColumnMetaData(final Map<String, Integer> dataTypeMap, final Set<String> primaryKeys, final ResultSet resultSet) throws SQLException {
+ String tableName = resultSet.getString("table_name");
+ String columnName = resultSet.getString("column_name");
+ String dataType = resultSet.getString("udt_name");
+ boolean isPrimaryKey = primaryKeys.contains(tableName + "," + columnName);
+ String columnDefault = resultSet.getString("column_default");
+ boolean generated = null != columnDefault && columnDefault.startsWith("nextval(");
+ //TODO user defined collation which deterministic is false
+ boolean caseSensitive = true;
+ return new ColumnMetaData(columnName, dataTypeMap.get(dataType), isPrimaryKey, generated, caseSensitive);
+ }
+
+ private String getTableMetaDataSQL(final Collection<String> existedTables) {
+ return existedTables.isEmpty() ? BASIC_TABLE_META_DATA_SQL
+ : String.format(TABLE_META_DATA_SQL_WITH_EXISTED_TABLES, existedTables.stream().map(each -> String.format("'%s'", each)).collect(Collectors.joining(",")));
+ }
+
+ private Map<String, Collection<IndexMetaData>> loadIndexMetaDataMap(final DataSource dataSource) throws SQLException {
+ Map<String, Collection<IndexMetaData>> result = new HashMap<>();
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement preparedStatement = connection.prepareStatement(BASIC_INDEX_META_DATA_SQL)) {
+ preparedStatement.setString(1, connection.getSchema());
+ try (ResultSet resultSet = preparedStatement.executeQuery()) {
+ while (resultSet.next()) {
+ String tableName = resultSet.getString("tablename");
+ Collection<IndexMetaData> indexes = result.computeIfAbsent(tableName, k -> new LinkedList<>());
+ String indexName = resultSet.getString("indexname");
+ indexes.add(new IndexMetaData(indexName));
+ }
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public String getDatabaseType() {
+ return "PostgreSQL";
+ }
+}
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/main/resources/META-INF/services/org.apache.shardingsphere.infra.metadata.schema.builder.spi.DialectTableMetaDataLoader b/shardingsphere-infra/shardingsphere-infra-common/src/main/resources/META-INF/services/org.apache.shardingsphere.infra.metadata.schema.builder.spi.DialectTableMetaDataLoader
index 18a9e83..7948e95 100644
--- a/shardingsphere-infra/shardingsphere-infra-common/src/main/resources/META-INF/services/org.apache.shardingsphere.infra.metadata.schema.builder.spi.DialectTableMetaDataLoader
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/main/resources/META-INF/services/org.apache.shardingsphere.infra.metadata.schema.builder.spi.DialectTableMetaDataLoader
@@ -17,3 +17,4 @@
org.apache.shardingsphere.infra.metadata.schema.builder.loader.dialect.MySQLTableMetaDataLoader
org.apache.shardingsphere.infra.metadata.schema.builder.loader.dialect.SQLServerTableMetaDataLoader
+org.apache.shardingsphere.infra.metadata.schema.builder.loader.dialect.PostgreSQLTableMetaDataLoader
diff --git a/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/metadata/schema/builder/loader/dialect/PostgreSQLTableMetaDataLoaderTest.java b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/metadata/schema/builder/loader/dialect/PostgreSQLTableMetaDataLoaderTest.java
new file mode 100644
index 0000000..7f3ee0a
--- /dev/null
+++ b/shardingsphere-infra/shardingsphere-infra-common/src/test/java/org/apache/shardingsphere/infra/metadata/schema/builder/loader/dialect/PostgreSQLTableMetaDataLoaderTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.infra.metadata.schema.builder.loader.dialect;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.Collections;
+import java.util.Map;
+import javax.sql.DataSource;
+import org.apache.shardingsphere.infra.metadata.schema.builder.spi.DialectTableMetaDataLoader;
+import org.apache.shardingsphere.infra.metadata.schema.model.ColumnMetaData;
+import org.apache.shardingsphere.infra.metadata.schema.model.IndexMetaData;
+import org.apache.shardingsphere.infra.metadata.schema.model.TableMetaData;
+import org.apache.shardingsphere.infra.spi.ShardingSphereServiceLoader;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public final class PostgreSQLTableMetaDataLoaderTest {
+
+ private static final String BASIC_TABLE_META_DATA_SQL = "SELECT table_name, column_name, data_type, udt_name, column_default FROM information_schema.columns WHERE table_schema = ?";
+
+ private static final String TABLE_META_DATA_SQL_WITH_EXISTED_TABLES = BASIC_TABLE_META_DATA_SQL + " AND table_name NOT IN ('existed_tbl')";
+
+ private static final String PRIMARY_KEY_META_DATA_SQL = "SELECT tc.table_name, kc.column_name FROM information_schema.table_constraints tc"
+ + " JOIN information_schema.key_column_usage kc"
+ + " ON kc.table_schema = tc.table_schema AND kc.table_name = tc.table_name AND kc.constraint_name = tc.constraint_name"
+ + " WHERE tc.constraint_type = 'PRIMARY KEY' AND kc.ordinal_position IS NOT NULL AND kc.table_schema = ?";
+
+ private static final String BASIC_INDEX_META_DATA_SQL = "SELECT tablename, indexname FROM pg_indexes WHERE schemaname = ?";
+
+ @BeforeClass
+ public static void setUp() {
+ ShardingSphereServiceLoader.register(DialectTableMetaDataLoader.class);
+ }
+
+ @Test
+ public void assertLoadWithoutExistedTables() throws SQLException {
+ DataSource dataSource = mockDataSource();
+ ResultSet resultSet = mockTableMetaDataResultSet();
+ when(dataSource.getConnection().prepareStatement(BASIC_TABLE_META_DATA_SQL).executeQuery()).thenReturn(resultSet);
+ ResultSet primaryKeyResultSet = mockPrimaryKeyMetaDataResultSet();
+ when(dataSource.getConnection().prepareStatement(PRIMARY_KEY_META_DATA_SQL).executeQuery()).thenReturn(primaryKeyResultSet);
+ ResultSet indexResultSet = mockIndexMetaDataResultSet();
+ when(dataSource.getConnection().prepareStatement(BASIC_INDEX_META_DATA_SQL).executeQuery()).thenReturn(indexResultSet);
+ assertTableMetaDataMap(getTableMetaDataLoader().load(dataSource, Collections.emptyList()));
+ }
+
+ @Test
+ public void assertLoadWithExistedTables() throws SQLException {
+ DataSource dataSource = mockDataSource();
+ ResultSet resultSet = mockTableMetaDataResultSet();
+ when(dataSource.getConnection().prepareStatement(TABLE_META_DATA_SQL_WITH_EXISTED_TABLES).executeQuery()).thenReturn(resultSet);
+ ResultSet primaryKeyResultSet = mockPrimaryKeyMetaDataResultSet();
+ when(dataSource.getConnection().prepareStatement(PRIMARY_KEY_META_DATA_SQL).executeQuery()).thenReturn(primaryKeyResultSet);
+ ResultSet indexResultSet = mockIndexMetaDataResultSet();
+ when(dataSource.getConnection().prepareStatement(BASIC_INDEX_META_DATA_SQL).executeQuery()).thenReturn(indexResultSet);
+ assertTableMetaDataMap(getTableMetaDataLoader().load(dataSource, Collections.singletonList("existed_tbl")));
+ }
+
+ private DataSource mockDataSource() throws SQLException {
+ DataSource result = mock(DataSource.class, RETURNS_DEEP_STUBS);
+ ResultSet typeInfoResultSet = mockTypeInfoResultSet();
+ when(result.getConnection().getMetaData().getTypeInfo()).thenReturn(typeInfoResultSet);
+ return result;
+ }
+
+ private ResultSet mockTypeInfoResultSet() throws SQLException {
+ ResultSet result = mock(ResultSet.class);
+ when(result.next()).thenReturn(true, true, false);
+ when(result.getString("TYPE_NAME")).thenReturn("int4", "varchar");
+ when(result.getInt("DATA_TYPE")).thenReturn(Types.INTEGER, Types.VARCHAR);
+ return result;
+ }
+
+ private ResultSet mockTableMetaDataResultSet() throws SQLException {
+ ResultSet result = mock(ResultSet.class);
+ when(result.next()).thenReturn(true, true, false);
+ when(result.getString("table_name")).thenReturn("tbl");
+ when(result.getString("column_name")).thenReturn("id", "name");
+ when(result.getString("data_type")).thenReturn("integer", "character varying");
+ when(result.getString("udt_name")).thenReturn("int4", "varchar");
+ when(result.getString("column_default")).thenReturn("nextval('id_seq'::regclass)", "");
+ return result;
+ }
+
+ private ResultSet mockPrimaryKeyMetaDataResultSet() throws SQLException {
+ ResultSet result = mock(ResultSet.class);
+ when(result.next()).thenReturn(true, false);
+ when(result.getString("table_name")).thenReturn("tbl");
+ when(result.getString("column_name")).thenReturn("id");
+ return result;
+ }
+
+ private ResultSet mockIndexMetaDataResultSet() throws SQLException {
+ ResultSet result = mock(ResultSet.class);
+ when(result.next()).thenReturn(true, false);
+ when(result.getString("tablename")).thenReturn("tbl");
+ when(result.getString("indexname")).thenReturn("id");
+ return result;
+ }
+
+ private DialectTableMetaDataLoader getTableMetaDataLoader() {
+ for (DialectTableMetaDataLoader each : ShardingSphereServiceLoader.newServiceInstances(DialectTableMetaDataLoader.class)) {
+ if ("PostgreSQL".equals(each.getDatabaseType())) {
+ return each;
+ }
+ }
+ throw new IllegalStateException("Can not find PostgreSQLTableMetaDataLoader");
+ }
+
+ private void assertTableMetaDataMap(final Map<String, TableMetaData> actual) {
+ assertThat(actual.size(), is(1));
+ assertThat(actual.get("tbl").getColumns().size(), is(2));
+ assertThat(actual.get("tbl").getColumnMetaData(0), is(new ColumnMetaData("id", Types.INTEGER, true, true, true)));
+ assertThat(actual.get("tbl").getColumnMetaData(1), is(new ColumnMetaData("name", Types.VARCHAR, false, false, true)));
+ assertThat(actual.get("tbl").getIndexes().size(), is(1));
+ assertThat(actual.get("tbl").getIndexes().get("id"), is(new IndexMetaData("id")));
+ }
+}