You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by du...@apache.org on 2023/05/12 08:31:43 UTC
[shardingsphere] branch master updated: Add NewMetaDataPersistService to write new data source metadata structure (#25604)
This is an automated email from the ASF dual-hosted git repository.
duanzhengqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 2df49e27627 Add NewMetaDataPersistService to write new data source metadata structure (#25604)
2df49e27627 is described below
commit 2df49e276277c62db533ce0fd133f41bf70d58ca
Author: zhaojinchao <zh...@apache.org>
AuthorDate: Fri May 12 16:31:34 2023 +0800
Add NewMetaDataPersistService to write new data source metadata structure (#25604)
* Add NewMetaDataPersistService to write new data source metadata structure.
* Update style
---
.../persist/NewMetaDataPersistService.java | 6 +-
.../persist/node/NewDatabaseMetaDataNode.java | 74 ++++++++++++++++++++
.../persist/node/metadata/DataSourceNode.java | 67 ++++++++++++++++++
.../config/database/DataSourcePersistService.java | 20 ++----
.../database/DatabaseBasedPersistService.java | 22 ------
.../database/DatabaseRulePersistService.java | 9 +--
.../database/NewDataSourcePersistService.java | 81 ++++++++++++++++++++++
.../database/DatabaseRulePersistServiceTest.java | 10 ---
8 files changed, 231 insertions(+), 58 deletions(-)
diff --git a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/NewMetaDataPersistService.java b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/NewMetaDataPersistService.java
index 359f55364af..b95ec4b7a74 100644
--- a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/NewMetaDataPersistService.java
+++ b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/NewMetaDataPersistService.java
@@ -26,9 +26,9 @@ import org.apache.shardingsphere.infra.datasource.props.DataSourceProperties;
import org.apache.shardingsphere.infra.datasource.props.DataSourcePropertiesCreator;
import org.apache.shardingsphere.infra.rule.ShardingSphereRule;
import org.apache.shardingsphere.metadata.persist.data.ShardingSphereDataPersistService;
+import org.apache.shardingsphere.metadata.persist.service.config.database.NewDataSourcePersistService;
import org.apache.shardingsphere.metadata.persist.service.database.DatabaseMetaDataPersistService;
import org.apache.shardingsphere.metadata.persist.service.version.MetaDataVersionPersistService;
-import org.apache.shardingsphere.metadata.persist.service.config.database.DataSourcePersistService;
import org.apache.shardingsphere.metadata.persist.service.config.database.DatabaseRulePersistService;
import org.apache.shardingsphere.metadata.persist.service.config.global.GlobalRulePersistService;
import org.apache.shardingsphere.metadata.persist.service.config.global.PropertiesPersistService;
@@ -51,7 +51,7 @@ public final class NewMetaDataPersistService implements MetaDataBasedPersistServ
private final PersistRepository repository;
// TODO replace all service to new persist service
- private final DataSourcePersistService dataSourceService;
+ private final NewDataSourcePersistService dataSourceService;
private final DatabaseMetaDataPersistService databaseMetaDataService;
@@ -67,7 +67,7 @@ public final class NewMetaDataPersistService implements MetaDataBasedPersistServ
public NewMetaDataPersistService(final PersistRepository repository) {
this.repository = repository;
- dataSourceService = new DataSourcePersistService(repository);
+ dataSourceService = new NewDataSourcePersistService(repository);
databaseMetaDataService = new DatabaseMetaDataPersistService(repository);
databaseRulePersistService = new DatabaseRulePersistService(repository);
globalRuleService = new GlobalRulePersistService(repository);
diff --git a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/node/NewDatabaseMetaDataNode.java b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/node/NewDatabaseMetaDataNode.java
new file mode 100644
index 00000000000..d2154f3562f
--- /dev/null
+++ b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/node/NewDatabaseMetaDataNode.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.metadata.persist.node;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import org.apache.shardingsphere.metadata.persist.node.metadata.DataSourceNode;
+
+/**
+ * TODO Rename DatabaseMetaDataNode when metadata structure adjustment completed. #25485
+ * New database meta data node.
+ */
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public final class NewDatabaseMetaDataNode {
+
+ private static final String ROOT_NODE = "metadata";
+
+ /**
+ * Get data Sources path.
+ *
+ * @param databaseName database name
+ * @return data sources path
+ */
+ public static String getDataSourcesPath(final String databaseName) {
+ return String.join("/", getMetaDataNodePath(), DataSourceNode.getDataSourcesPath(databaseName));
+ }
+
+ /**
+ * Get data Source path.
+ *
+ * @param databaseName database name
+ * @param dataSourceName data source name
+ * @param version version
+ * @return data source path
+ */
+ public static String getDataSourcePath(final String databaseName, final String dataSourceName, final String version) {
+ return String.join("/", getMetaDataNodePath(), DataSourceNode.getDataSourcePath(databaseName, dataSourceName, version));
+ }
+
+ /**
+ * Get data Source active version path.
+ *
+ * @param databaseName database name
+ * @param dataSourceName data source name
+ * @return data source active version path
+ */
+ public static String getDataSourceActiveVersionPath(final String databaseName, final String dataSourceName) {
+ return String.join("/", getMetaDataNodePath(), DataSourceNode.getActiveVersionPath(databaseName, dataSourceName));
+ }
+
+ /**
+ * Get meta data node path.
+ *
+ * @return meta data node path
+ */
+ public static String getMetaDataNodePath() {
+ return String.join("/", "", ROOT_NODE);
+ }
+}
diff --git a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/node/metadata/DataSourceNode.java b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/node/metadata/DataSourceNode.java
new file mode 100644
index 00000000000..660f932f894
--- /dev/null
+++ b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/node/metadata/DataSourceNode.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.metadata.persist.node.metadata;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+/**
+ * Database source node.
+ */
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public final class DataSourceNode {
+
+ private static final String ROOT_NODE = "data_sources";
+
+ private static final String ACTIVE_VERSION = "active_version";
+
+ private static final String VERSIONS = "versions";
+
+ /**
+ * Get data sources path.
+ *
+ * @param databaseName database name
+ * @return data sources path
+ */
+ public static String getDataSourcesPath(final String databaseName) {
+ return String.join("/", databaseName, ROOT_NODE);
+ }
+
+ /**
+ * Get data source path.
+ *
+ * @param databaseName database name
+ * @param dataSourceName data source name
+ * @param version data source version
+ * @return data source path
+ */
+ public static String getDataSourcePath(final String databaseName, final String dataSourceName, final String version) {
+ return String.join("/", databaseName, ROOT_NODE, dataSourceName, VERSIONS, version);
+ }
+
+ /**
+ * Get data source path.
+ *
+ * @param databaseName database name
+ * @param dataSourceName data source name
+ * @return active version path
+ */
+ public static String getActiveVersionPath(final String databaseName, final String dataSourceName) {
+ return String.join("/", databaseName, ROOT_NODE, dataSourceName, ACTIVE_VERSION);
+ }
+}
diff --git a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DataSourcePersistService.java b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DataSourcePersistService.java
index 1f07cfdd0c0..ec9d391dd07 100644
--- a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DataSourcePersistService.java
+++ b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DataSourcePersistService.java
@@ -20,14 +20,11 @@ package org.apache.shardingsphere.metadata.persist.service.config.database;
import com.google.common.base.Strings;
import lombok.RequiredArgsConstructor;
import org.apache.shardingsphere.infra.datasource.props.DataSourceProperties;
-import org.apache.shardingsphere.infra.rule.ShardingSphereRule;
import org.apache.shardingsphere.infra.util.yaml.YamlEngine;
import org.apache.shardingsphere.infra.yaml.config.swapper.resource.YamlDataSourceConfigurationSwapper;
import org.apache.shardingsphere.metadata.persist.node.DatabaseMetaDataNode;
import org.apache.shardingsphere.mode.spi.PersistRepository;
-import javax.sql.DataSource;
-import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -52,12 +49,6 @@ public final class DataSourcePersistService implements DatabaseBasedPersistServi
YamlEngine.marshal(swapYamlDataSourceConfiguration(dataSourceConfigs)));
}
- @Override
- public void persist(final String databaseName, final String version, final Map<String, DataSource> dataSources,
- final Collection<ShardingSphereRule> rules, final Map<String, DataSourceProperties> dataSourcePropsMap) {
- repository.persist(DatabaseMetaDataNode.getMetaDataDataSourcePath(databaseName, version), YamlEngine.marshal(swapYamlDataSourceConfiguration(dataSourcePropsMap)));
- }
-
private Map<String, Map<String, Object>> swapYamlDataSourceConfiguration(final Map<String, DataSourceProperties> dataSourcePropsMap) {
return dataSourcePropsMap.entrySet().stream()
.collect(Collectors.toMap(Entry::getKey, entry -> new YamlDataSourceConfigurationSwapper().swapToMap(entry.getValue()), (oldValue, currentValue) -> oldValue, LinkedHashMap::new));
@@ -75,6 +66,11 @@ public final class DataSourcePersistService implements DatabaseBasedPersistServi
return Strings.isNullOrEmpty(yamlContent) ? new LinkedHashMap<>() : getDataSourceProperties(yamlContent);
}
+ private boolean isExisted(final String databaseName) {
+ return !Strings.isNullOrEmpty(getDatabaseActiveVersion(databaseName)) && !Strings.isNullOrEmpty(repository.getDirectly(DatabaseMetaDataNode.getMetaDataDataSourcePath(databaseName,
+ getDatabaseActiveVersion(databaseName))));
+ }
+
@SuppressWarnings("unchecked")
private Map<String, DataSourceProperties> getDataSourceProperties(final String yamlContent) {
Map<String, Map<String, Object>> yamlDataSources = YamlEngine.unmarshal(yamlContent, Map.class);
@@ -86,12 +82,6 @@ public final class DataSourcePersistService implements DatabaseBasedPersistServi
return result;
}
- @Override
- public boolean isExisted(final String databaseName) {
- return !Strings.isNullOrEmpty(getDatabaseActiveVersion(databaseName)) && !Strings.isNullOrEmpty(repository.getDirectly(DatabaseMetaDataNode.getMetaDataDataSourcePath(databaseName,
- getDatabaseActiveVersion(databaseName))));
- }
-
/**
* Append data source properties map.
*
diff --git a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseBasedPersistService.java b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseBasedPersistService.java
index 3cdb0d968a4..d39729706a4 100644
--- a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseBasedPersistService.java
+++ b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseBasedPersistService.java
@@ -18,11 +18,8 @@
package org.apache.shardingsphere.metadata.persist.service.config.database;
import org.apache.shardingsphere.infra.datasource.props.DataSourceProperties;
-import org.apache.shardingsphere.infra.rule.ShardingSphereRule;
-import javax.sql.DataSource;
import java.util.Map;
-import java.util.Collection;
/**
* Database based persist service.
@@ -39,17 +36,6 @@ public interface DatabaseBasedPersistService<T> {
*/
void persist(String databaseName, T configs);
- /**
- * Persist version configurations.
- *
- * @param databaseName database name
- * @param version version
- * @param dataSources data sources
- * @param rules rules
- * @param configs configurations
- */
- void persist(String databaseName, String version, Map<String, DataSource> dataSources, Collection<ShardingSphereRule> rules, T configs);
-
/**
* Load configurations.
*
@@ -67,14 +53,6 @@ public interface DatabaseBasedPersistService<T> {
*/
T load(String databaseName, String version);
- /**
- * Judge whether schema configuration existed.
- *
- * @param databaseName database name
- * @return configuration existed or not
- */
- boolean isExisted(String databaseName);
-
/**
* TODO remove this after meta data refactor completed
* Append data source properties map.
diff --git a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistService.java b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistService.java
index 1de8ac02629..8502e2df664 100644
--- a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistService.java
+++ b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistService.java
@@ -61,12 +61,6 @@ public final class DatabaseRulePersistService implements DatabaseRuleBasedPersis
YamlEngine.marshal(createYamlRuleConfigurations(configs)));
}
- @Override
- public void persist(final String databaseName, final String version, final Map<String, DataSource> dataSources,
- final Collection<ShardingSphereRule> rules, final Collection<RuleConfiguration> configs) {
- repository.persist(DatabaseMetaDataNode.getRulePath(databaseName, version), YamlEngine.marshal(createYamlRuleConfigurations(dataSources, rules, configs)));
- }
-
private Collection<YamlRuleConfiguration> createYamlRuleConfigurations(final Collection<RuleConfiguration> ruleConfigs) {
return new YamlRuleConfigurationSwapperEngine().swapToYamlRuleConfigurations(ruleConfigs);
}
@@ -95,8 +89,7 @@ public final class DatabaseRulePersistService implements DatabaseRuleBasedPersis
.getRulePath(databaseName, getDatabaseActiveVersion(databaseName))), Collection.class, true));
}
- @Override
- public boolean isExisted(final String databaseName) {
+ private boolean isExisted(final String databaseName) {
return !Strings.isNullOrEmpty(getDatabaseActiveVersion(databaseName))
&& !Strings.isNullOrEmpty(repository.getDirectly(DatabaseMetaDataNode.getRulePath(databaseName, getDatabaseActiveVersion(databaseName))));
}
diff --git a/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/NewDataSourcePersistService.java b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/NewDataSourcePersistService.java
new file mode 100644
index 00000000000..479b697be74
--- /dev/null
+++ b/kernel/metadata/core/src/main/java/org/apache/shardingsphere/metadata/persist/service/config/database/NewDataSourcePersistService.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.metadata.persist.service.config.database;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import lombok.RequiredArgsConstructor;
+import org.apache.shardingsphere.infra.datasource.props.DataSourceProperties;
+import org.apache.shardingsphere.infra.util.yaml.YamlEngine;
+import org.apache.shardingsphere.infra.yaml.config.swapper.resource.YamlDataSourceConfigurationSwapper;
+import org.apache.shardingsphere.metadata.persist.node.NewDatabaseMetaDataNode;
+import org.apache.shardingsphere.mode.spi.PersistRepository;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+
+/**
+ * TODO Rename DataSourcePersistService when metadata structure adjustment completed. #25485
+ * New Data source persist service.
+ */
+@RequiredArgsConstructor
+public final class NewDataSourcePersistService implements DatabaseBasedPersistService<Map<String, DataSourceProperties>> {
+
+ private static final String DEFAULT_VERSION = "0";
+
+ private final PersistRepository repository;
+
+ @Override
+ public void persist(final String databaseName, final Map<String, DataSourceProperties> dataSourceConfigs) {
+ for (Entry<String, DataSourceProperties> entry : dataSourceConfigs.entrySet()) {
+ String activeVersion = getDatabaseActiveVersion(databaseName, entry.getKey());
+ if (Strings.isNullOrEmpty(activeVersion)) {
+ repository.persist(NewDatabaseMetaDataNode.getDataSourceActiveVersionPath(databaseName, entry.getKey()), DEFAULT_VERSION);
+ }
+ repository.persist(NewDatabaseMetaDataNode.getDataSourcePath(databaseName, entry.getKey(), DEFAULT_VERSION),
+ YamlEngine.marshal(new YamlDataSourceConfigurationSwapper().swapToMap(entry.getValue())));
+ }
+ }
+
+ @Override
+ public Map<String, DataSourceProperties> load(final String databaseName) {
+ Map<String, DataSourceProperties> result = new LinkedHashMap<>();
+ for (String each : repository.getChildrenKeys(NewDatabaseMetaDataNode.getDataSourcesPath(databaseName))) {
+ result.put(each, getDataSourceProps(databaseName, each));
+ }
+ return result;
+ }
+
+ // TODO Remove this
+ @Override
+ public Map<String, DataSourceProperties> load(final String databaseName, final String version) {
+ return Collections.emptyMap();
+ }
+
+ private DataSourceProperties getDataSourceProps(final String databaseName, final String dataSourceName) {
+ String result = repository.getDirectly(NewDatabaseMetaDataNode.getDataSourcePath(databaseName, getDatabaseActiveVersion(databaseName, dataSourceName), dataSourceName));
+ Preconditions.checkState(!Strings.isNullOrEmpty(result), "Not found `%s` data source config in `%s` database", dataSourceName, databaseName);
+ return YamlEngine.unmarshal(result, DataSourceProperties.class);
+ }
+
+ private String getDatabaseActiveVersion(final String databaseName, final String dataSourceName) {
+ return repository.getDirectly(NewDatabaseMetaDataNode.getDataSourceActiveVersionPath(databaseName, dataSourceName));
+ }
+}
diff --git a/kernel/metadata/core/src/test/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistServiceTest.java b/kernel/metadata/core/src/test/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistServiceTest.java
index 4a40542193a..7c00a07af65 100644
--- a/kernel/metadata/core/src/test/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistServiceTest.java
+++ b/kernel/metadata/core/src/test/java/org/apache/shardingsphere/metadata/persist/service/config/database/DatabaseRulePersistServiceTest.java
@@ -34,7 +34,6 @@ import java.util.stream.Collectors;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.when;
@@ -57,15 +56,6 @@ class DatabaseRulePersistServiceTest {
assertThat(actual.size(), is(1));
}
- @Test
- void assertIsExisted() {
- when(repository.getDirectly("/metadata/foo_db/active_version")).thenReturn("0");
- when(repository.getDirectly("/metadata/foo_db/versions/0/rules")).thenReturn(readYAML());
- DatabaseRulePersistService databaseRulePersistService = new DatabaseRulePersistService(repository);
- assertTrue(databaseRulePersistService.isExisted("foo_db"));
- assertFalse(databaseRulePersistService.isExisted("foo_db_1"));
- }
-
@SneakyThrows({IOException.class, URISyntaxException.class})
private String readYAML() {
return Files.readAllLines(Paths.get(ClassLoader.getSystemResource("yaml/persist/data-database-rule.yaml").toURI()))