You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by me...@apache.org on 2021/02/26 09:59:46 UTC
[shardingsphere] branch master updated: Replace
YamlDataSourceRuleConfigurationWrap with YamlRootRuleConfigurations (#9517)
This is an automated email from the ASF dual-hosted git repository.
menghaoran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 1633bf2 Replace YamlDataSourceRuleConfigurationWrap with YamlRootRuleConfigurations (#9517)
1633bf2 is described below
commit 1633bf249e9ebced1bc38273f07e98b1f4a743c3
Author: 邱鹿 Lucas <lu...@163.com>
AuthorDate: Fri Feb 26 17:59:20 2021 +0800
Replace YamlDataSourceRuleConfigurationWrap with YamlRootRuleConfigurations (#9517)
* replace YamlDataSourceRuleConfigurationWrap with YamlRootRuleConfigurations
* add Governance config check
Co-authored-by: qiulu3 <Lucas209910>
---
.../yaml/config/YamlConfigurationConverter.java | 5 +--
.../YamlDataSourceRuleConfigurationWrap.java | 39 ----------------------
.../util/ServerConfigurationInitializer.java | 1 +
.../scaling/core/api/impl/ScalingAPIImpl.java | 3 +-
.../ShardingSphereJDBCDataSourceConfiguration.java | 15 +++++----
.../shardingsphere/scaling/core/util/JDBCUtil.java | 15 ++++++++-
.../scaling/core/util/JobConfigurationUtil.java | 29 ++++++++--------
.../scaling/core/util/JDBCUtilTest.java | 2 +-
8 files changed, 41 insertions(+), 68 deletions(-)
diff --git a/shardingsphere-governance/shardingsphere-governance-core/src/main/java/org/apache/shardingsphere/governance/core/yaml/config/YamlConfigurationConverter.java b/shardingsphere-governance/shardingsphere-governance-core/src/main/java/org/apache/shardingsphere/governance/core/yaml/config/YamlConfigurationConverter.java
index 23e0e27..14e9e8a 100644
--- a/shardingsphere-governance/shardingsphere-governance-core/src/main/java/org/apache/shardingsphere/governance/core/yaml/config/YamlConfigurationConverter.java
+++ b/shardingsphere-governance/shardingsphere-governance-core/src/main/java/org/apache/shardingsphere/governance/core/yaml/config/YamlConfigurationConverter.java
@@ -38,10 +38,8 @@ import javax.sql.DataSource;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Optional;
import java.util.Properties;
-import java.util.stream.Collectors;
/**
* Configuration converter for YAML content.
@@ -60,8 +58,7 @@ public final class YamlConfigurationConverter {
if (null == result.getDataSources() || result.getDataSources().isEmpty()) {
return new LinkedHashMap<>();
}
- return result.getDataSources().entrySet().stream().collect(Collectors.toMap(Entry::getKey, entry -> new YamlDataSourceConfigurationSwapper()
- .swapToObjectFromMap(entry.getValue()), (oldValue, currentValue) -> oldValue, LinkedHashMap::new));
+ return convertDataSourceConfigurations(result.getDataSources());
}
/**
diff --git a/shardingsphere-governance/shardingsphere-governance-core/src/main/java/org/apache/shardingsphere/governance/core/yaml/config/YamlDataSourceRuleConfigurationWrap.java b/shardingsphere-governance/shardingsphere-governance-core/src/main/java/org/apache/shardingsphere/governance/core/yaml/config/YamlDataSourceRuleConfigurationWrap.java
deleted file mode 100644
index 4bca368..0000000
--- a/shardingsphere-governance/shardingsphere-governance-core/src/main/java/org/apache/shardingsphere/governance/core/yaml/config/YamlDataSourceRuleConfigurationWrap.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.shardingsphere.governance.core.yaml.config;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.apache.shardingsphere.infra.yaml.config.YamlConfiguration;
-import org.apache.shardingsphere.infra.yaml.config.YamlRuleConfiguration;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.Map;
-
-/**
- * Yaml data source and rule configuration wrap.
- */
-@Getter
-@Setter
-public final class YamlDataSourceRuleConfigurationWrap implements YamlConfiguration {
-
- private Map<String, Map<String, Object>> dataSources;
-
- private Collection<YamlRuleConfiguration> rules = new LinkedList<>();
-}
diff --git a/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/main/java/org/apache/shardingsphere/scaling/util/ServerConfigurationInitializer.java b/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/main/java/org/apache/shardingsphere/scaling/util/ServerConfigurationInitializer.java
index 08353ad..70d2b24 100644
--- a/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/main/java/org/apache/shardingsphere/scaling/util/ServerConfigurationInitializer.java
+++ b/shardingsphere-scaling/shardingsphere-scaling-bootstrap/src/main/java/org/apache/shardingsphere/scaling/util/ServerConfigurationInitializer.java
@@ -49,6 +49,7 @@ public final class ServerConfigurationInitializer {
File yamlFile = new File(Resources.getResource(SERVER_FILE).getPath());
YamlServerConfiguration serverConfig = YamlEngine.unmarshal(yamlFile, YamlServerConfiguration.class);
Preconditions.checkNotNull(serverConfig, "Server configuration file `%s` is invalid.", yamlFile.getName());
+ Preconditions.checkNotNull(serverConfig.getGovernance(), "Governance configuration is required.");
ScalingContext.getInstance().init(new ServerConfigurationYamlSwapper().swapToObject(serverConfig));
}
}
diff --git a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingAPIImpl.java b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingAPIImpl.java
index 09d0666..cd2f64e 100644
--- a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingAPIImpl.java
+++ b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingAPIImpl.java
@@ -144,8 +144,7 @@ public final class ScalingAPIImpl implements ScalingAPI {
try {
return ScalingAPIFactory.getJobConfigurationAPI().getJobConfiguration(String.valueOf(jobId));
} catch (final NullPointerException ex) {
- log.warn("Get job {} config failed.", jobId);
- throw new ScalingJobNotFoundException(String.format("Can not find job by id %s", jobId));
+ throw new ScalingJobNotFoundException(String.format("Can not find scaling job %s", jobId));
}
}
}
diff --git a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/config/datasource/ShardingSphereJDBCDataSourceConfiguration.java b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/config/datasource/ShardingSphereJDBCDataSourceConfiguration.java
index cc74674..2f9ecbb 100644
--- a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/config/datasource/ShardingSphereJDBCDataSourceConfiguration.java
+++ b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/config/datasource/ShardingSphereJDBCDataSourceConfiguration.java
@@ -22,10 +22,11 @@ import lombok.EqualsAndHashCode;
import lombok.Getter;
import org.apache.shardingsphere.driver.api.ShardingSphereDataSourceFactory;
import org.apache.shardingsphere.governance.core.yaml.config.YamlConfigurationConverter;
-import org.apache.shardingsphere.governance.core.yaml.config.YamlDataSourceRuleConfigurationWrap;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.DatabaseTypeRegistry;
+import org.apache.shardingsphere.infra.yaml.config.YamlRootRuleConfigurations;
import org.apache.shardingsphere.infra.yaml.engine.YamlEngine;
+import org.apache.shardingsphere.scaling.core.util.JDBCUtil;
import javax.sql.DataSource;
import java.sql.SQLException;
@@ -45,15 +46,15 @@ public final class ShardingSphereJDBCDataSourceConfiguration implements ScalingD
private final String parameter;
- private final YamlDataSourceRuleConfigurationWrap dataSourceRuleConfig;
+ private final YamlRootRuleConfigurations rootRuleConfigs;
private final DatabaseType databaseType;
public ShardingSphereJDBCDataSourceConfiguration(final String parameter) {
this.parameter = parameter;
- dataSourceRuleConfig = YamlEngine.unmarshal(parameter, YamlDataSourceRuleConfigurationWrap.class);
- Map<String, Object> props = dataSourceRuleConfig.getDataSources().values().iterator().next();
- databaseType = DatabaseTypeRegistry.getDatabaseTypeByURL(props.getOrDefault("url", props.get("jdbcUrl")).toString());
+ rootRuleConfigs = YamlEngine.unmarshal(parameter, YamlRootRuleConfigurations.class);
+ Map<String, Object> props = rootRuleConfigs.getDataSources().values().iterator().next();
+ databaseType = DatabaseTypeRegistry.getDatabaseTypeByURL(JDBCUtil.getJdbcUrl(props));
}
public ShardingSphereJDBCDataSourceConfiguration(final String dataSources, final String rules) {
@@ -70,7 +71,7 @@ public final class ShardingSphereJDBCDataSourceConfiguration implements ScalingD
@Override
public DataSource toDataSource() throws SQLException {
- return ShardingSphereDataSourceFactory.createDataSource(YamlConfigurationConverter.convertDataSources(dataSourceRuleConfig.getDataSources()),
- Lists.newArrayList(YamlConfigurationConverter.convertShardingRuleConfig(dataSourceRuleConfig.getRules())), null);
+ return ShardingSphereDataSourceFactory.createDataSource(YamlConfigurationConverter.convertDataSources(rootRuleConfigs.getDataSources()),
+ Lists.newArrayList(YamlConfigurationConverter.convertShardingRuleConfig(rootRuleConfigs.getRules())), null);
}
}
diff --git a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JDBCUtil.java b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JDBCUtil.java
index 53242ee..d5745ae 100644
--- a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JDBCUtil.java
+++ b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JDBCUtil.java
@@ -17,6 +17,7 @@
package org.apache.shardingsphere.scaling.core.util;
+import com.google.common.base.Preconditions;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.shardingsphere.scaling.core.common.datasource.JdbcUri;
@@ -51,7 +52,7 @@ public final class JDBCUtil {
}
private static void append(final ShardingSphereJDBCDataSourceConfiguration dataSourceConfig, final Map<String, String> parameters) {
- dataSourceConfig.getDataSourceRuleConfig().getDataSources()
+ dataSourceConfig.getRootRuleConfigs().getDataSources()
.forEach((key, value) -> {
String jdbcUrlKey = value.containsKey("url") ? "url" : "jdbcUrl";
value.replace(jdbcUrlKey, append(value.get(jdbcUrlKey).toString(), parameters));
@@ -80,4 +81,16 @@ public final class JDBCUtil {
result.deleteCharAt(result.length() - 1);
return result.toString();
}
+
+ /**
+ * Get jdbc url from parameters, the key can be url or jdbcUrl.
+ *
+ * @param parameters parameters
+ * @return jdbc url
+ */
+ public static String getJdbcUrl(final Map<String, Object> parameters) {
+ Object result = parameters.getOrDefault("url", parameters.get("jdbcUrl"));
+ Preconditions.checkNotNull(result, "url or jdbcUrl is required.");
+ return result.toString();
+ }
}
diff --git a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JobConfigurationUtil.java b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JobConfigurationUtil.java
index 587cbb9..912ae1f 100644
--- a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JobConfigurationUtil.java
+++ b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/util/JobConfigurationUtil.java
@@ -27,8 +27,8 @@ import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.shardingsphere.governance.core.yaml.config.YamlConfigurationConverter;
-import org.apache.shardingsphere.governance.core.yaml.config.YamlDataSourceRuleConfigurationWrap;
import org.apache.shardingsphere.infra.config.datasource.DataSourceConfiguration;
+import org.apache.shardingsphere.infra.yaml.config.YamlRootRuleConfigurations;
import org.apache.shardingsphere.infra.yaml.engine.YamlEngine;
import org.apache.shardingsphere.scaling.core.common.datasource.JdbcUri;
import org.apache.shardingsphere.scaling.core.config.DumperConfiguration;
@@ -107,12 +107,12 @@ public final class JobConfigurationUtil {
"Only ShardingSphereJdbc type of source ScalingDataSourceConfiguration is supported.");
ShardingSphereJDBCDataSourceConfiguration source = (ShardingSphereJDBCDataSourceConfiguration) sourceConfig;
if (!(jobConfig.getRuleConfig().getTarget().unwrap() instanceof ShardingSphereJDBCDataSourceConfiguration)) {
- return getShardingRuleConfigMap(source.getDataSourceRuleConfig()).entrySet().stream()
+ return getShardingRuleConfigMap(source.getRootRuleConfigs()).entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, each -> each.getValue().getActualDataNodes()));
}
ShardingSphereJDBCDataSourceConfiguration target = (ShardingSphereJDBCDataSourceConfiguration) jobConfig.getRuleConfig().getTarget().unwrap();
- return getShouldScalingActualDataNodes(getModifiedDataSources(source.getDataSourceRuleConfig(), target.getDataSourceRuleConfig()),
- getShardingRuleConfigMap(source.getDataSourceRuleConfig()), getShardingRuleConfigMap(target.getDataSourceRuleConfig()));
+ return getShouldScalingActualDataNodes(getModifiedDataSources(source.getRootRuleConfigs(), target.getRootRuleConfigs()),
+ getShardingRuleConfigMap(source.getRootRuleConfigs()), getShardingRuleConfigMap(target.getRootRuleConfigs()));
}
private static Map<String, String> getShouldScalingActualDataNodes(final Set<String> modifiedDataSources,
@@ -132,10 +132,10 @@ public final class JobConfigurationUtil {
return result;
}
- private static Set<String> getModifiedDataSources(final YamlDataSourceRuleConfigurationWrap sourceDataSourceRuleConfig, final YamlDataSourceRuleConfigurationWrap targetDataSourceRuleConfig) {
+ private static Set<String> getModifiedDataSources(final YamlRootRuleConfigurations sourceRootRuleConfigs, final YamlRootRuleConfigurations targetRootRuleConfigs) {
Set<String> result = new HashSet<>();
- Map<String, String> oldDataSourceUrlMap = getDataSourceUrlMap(sourceDataSourceRuleConfig.getDataSources());
- Map<String, String> newDataSourceUrlMap = getDataSourceUrlMap(targetDataSourceRuleConfig.getDataSources());
+ Map<String, String> oldDataSourceUrlMap = getDataSourceUrlMap(sourceRootRuleConfigs.getDataSources());
+ Map<String, String> newDataSourceUrlMap = getDataSourceUrlMap(targetRootRuleConfigs.getDataSources());
newDataSourceUrlMap.forEach((key, value) -> {
if (!value.equals(oldDataSourceUrlMap.get(key))) {
result.add(key);
@@ -147,7 +147,7 @@ public final class JobConfigurationUtil {
private static Map<String, String> getDataSourceUrlMap(final Map<String, Map<String, Object>> dataSources) {
return dataSources.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> {
- JdbcUri uri = new JdbcUri(entry.getValue().getOrDefault("url", entry.getValue().get("jdbcUrl")).toString());
+ JdbcUri uri = new JdbcUri(JDBCUtil.getJdbcUrl(entry.getValue()));
return String.format("%s/%s", uri.getHost(), uri.getDatabase());
}));
}
@@ -156,9 +156,10 @@ public final class JobConfigurationUtil {
return actualDataNodes.stream().anyMatch(each -> modifiedDataSources.contains(each.split("\\.")[0]));
}
- private static Map<String, ShardingTableRuleConfiguration> getShardingRuleConfigMap(final YamlDataSourceRuleConfigurationWrap dataSourceRuleConfig) {
- ShardingRuleConfiguration ruleConfig = YamlConfigurationConverter.convertShardingRuleConfig(dataSourceRuleConfig.getRules());
- return ruleConfig.getTables().stream().collect(Collectors.toMap(ShardingTableRuleConfiguration::getLogicTable, Function.identity()));
+ private static Map<String, ShardingTableRuleConfiguration> getShardingRuleConfigMap(final YamlRootRuleConfigurations rootRuleConfigurations) {
+ ShardingRuleConfiguration ruleConfig = YamlConfigurationConverter.convertShardingRuleConfig(rootRuleConfigurations.getRules());
+ return ruleConfig.getTables().stream()
+ .collect(Collectors.toMap(ShardingTableRuleConfiguration::getLogicTable, Function.identity()));
}
private static String[] groupByDataSource(final Collection<String> actualDataNodeList) {
@@ -218,8 +219,8 @@ public final class JobConfigurationUtil {
public static List<TaskConfiguration> toTaskConfigs(final JobConfiguration jobConfig) {
List<TaskConfiguration> result = new LinkedList<>();
ShardingSphereJDBCDataSourceConfiguration sourceConfig = getSourceConfig(jobConfig);
- ShardingRuleConfiguration sourceRuleConfig = YamlConfigurationConverter.convertShardingRuleConfig(sourceConfig.getDataSourceRuleConfig().getRules());
- Map<String, DataSourceConfiguration> sourceDataSource = YamlConfigurationConverter.convertDataSourceConfigurations(sourceConfig.getDataSourceRuleConfig().getDataSources());
+ ShardingRuleConfiguration sourceRuleConfig = YamlConfigurationConverter.convertShardingRuleConfig(sourceConfig.getRootRuleConfigs().getRules());
+ Map<String, DataSourceConfiguration> sourceDataSource = YamlConfigurationConverter.convertDataSourceConfigurations(sourceConfig.getRootRuleConfigs().getDataSources());
Map<String, DataSource> dataSourceMap = sourceDataSource.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().createDataSource()));
Map<String, Map<String, String>> dataSourceTableNameMap = toDataSourceTableNameMap(new ShardingRule(sourceRuleConfig, sourceConfig.getDatabaseType(), dataSourceMap));
Optional<ShardingRuleConfiguration> targetRuleConfig = getTargetRuleConfig(jobConfig);
@@ -242,7 +243,7 @@ public final class JobConfigurationUtil {
private static Optional<ShardingRuleConfiguration> getTargetRuleConfig(final JobConfiguration jobConfig) {
ScalingDataSourceConfiguration dataSourceConfig = jobConfig.getRuleConfig().getTarget().unwrap();
if (dataSourceConfig instanceof ShardingSphereJDBCDataSourceConfiguration) {
- return Optional.of(YamlConfigurationConverter.convertShardingRuleConfig(((ShardingSphereJDBCDataSourceConfiguration) dataSourceConfig).getDataSourceRuleConfig().getRules()));
+ return Optional.of(YamlConfigurationConverter.convertShardingRuleConfig(((ShardingSphereJDBCDataSourceConfiguration) dataSourceConfig).getRootRuleConfigs().getRules()));
}
return Optional.empty();
}
diff --git a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/JDBCUtilTest.java b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/JDBCUtilTest.java
index 09fd32f..ec026da 100644
--- a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/JDBCUtilTest.java
+++ b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/util/JDBCUtilTest.java
@@ -42,7 +42,7 @@ public final class JDBCUtilTest {
public void assertAppendShardingSphereJDBCDataSourceConfig() {
ShardingSphereJDBCDataSourceConfiguration dataSourceConfig = new ShardingSphereJDBCDataSourceConfiguration(mockDataSource(), "");
JDBCUtil.appendJDBCParameter(dataSourceConfig, ImmutableMap.<String, String>builder().put("rewriteBatchedStatements", "true").build());
- ArrayList<DataSourceConfiguration> actual = new ArrayList<>(YamlConfigurationConverter.convertDataSourceConfigurations(dataSourceConfig.getDataSourceRuleConfig().getDataSources()).values());
+ ArrayList<DataSourceConfiguration> actual = new ArrayList<>(YamlConfigurationConverter.convertDataSourceConfigurations(dataSourceConfig.getRootRuleConfigs().getDataSources()).values());
assertThat(actual.get(0).getProps().get("url"), is("jdbc:mysql://192.168.0.2:3306/scaling?rewriteBatchedStatements=true&serverTimezone=UTC&useSSL=false"));
assertThat(actual.get(1).getProps().get("url"), is("jdbc:mysql://192.168.0.1:3306/scaling?rewriteBatchedStatements=true&serverTimezone=UTC&useSSL=false"));
}