You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by ji...@apache.org on 2023/06/28 06:11:36 UTC
[shardingsphere] branch master updated: Refactor DistSQL export/import metadata support broadcast & single (#26640)
This is an automated email from the ASF dual-hosted git repository.
jianglongtao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 7b07f47d44f Refactor DistSQL export/import metadata support broadcast & single (#26640)
7b07f47d44f is described below
commit 7b07f47d44f4fbe865e3215dcead04e96be67771
Author: ChenJiaHao <Pa...@163.com>
AuthorDate: Wed Jun 28 14:11:30 2023 +0800
Refactor DistSQL export/import metadata support broadcast & single (#26640)
* Refactor DistSQL export/import metadata support broadcast & single
* Fix UT
---
.../ral/queryable/ExportMetaDataExecutor.java | 6 ++--
.../ral/updatable/ImportMetaDataUpdater.java | 2 +-
.../YamlDatabaseConfigurationImportExecutor.java | 32 ++++++++++++++++++++++
.../ral/updatable/ImportMetaDataUpdaterTest.java | 2 +-
.../test/resources/conf/import/empty-metadata.data | 1 -
.../test/resources/conf/import/empty-metadata.json | 1 +
6 files changed, 38 insertions(+), 6 deletions(-)
diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
index 885dede4023..d1ebbc235a8 100644
--- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
+++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/queryable/ExportMetaDataExecutor.java
@@ -65,8 +65,8 @@ public final class ExportMetaDataExecutor implements MetaDataRequiredQueryableRA
return Collections.singleton(new LocalDataQueryResultRow(ProxyContext.getInstance().getContextManager().getInstanceContext().getInstance().getCurrentInstanceId(), LocalDateTime.now(),
String.format("Successfully exported to：'%s'", filePath)));
}
- return Collections.singleton(
- new LocalDataQueryResultRow(ProxyContext.getInstance().getContextManager().getInstanceContext().getInstance().getCurrentInstanceId(), LocalDateTime.now(), exportedData));
+ return Collections.singleton(new LocalDataQueryResultRow(
+ ProxyContext.getInstance().getContextManager().getInstanceContext().getInstance().getCurrentInstanceId(), LocalDateTime.now(), Base64.encodeBase64String(exportedData.getBytes())));
}
private String generateExportData(final ShardingSphereMetaData metaData) {
@@ -78,7 +78,7 @@ public final class ExportMetaDataExecutor implements MetaDataRequiredQueryableRA
ExportedClusterInfo exportedClusterInfo = new ExportedClusterInfo();
exportedClusterInfo.setMetaData(exportedMetaData);
generateSnapshotInfo(metaData, exportedClusterInfo);
- return Base64.encodeBase64String(JsonUtils.toJsonString(exportedClusterInfo).getBytes());
+ return JsonUtils.toJsonString(exportedClusterInfo);
}
private Map<String, String> getDatabases(final ProxyContext proxyContext) {
diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java
index 8ccc7731baf..d0666dfd1a2 100644
--- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java
+++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdater.java
@@ -54,7 +54,7 @@ public final class ImportMetaDataUpdater implements RALUpdater<ImportMetaDataSta
if (sqlStatement.getFilePath().isPresent()) {
File file = new File(sqlStatement.getFilePath().get());
try {
- jsonMetaDataConfig = new String(Base64.decodeBase64(FileUtils.readFileToString(file, Charset.defaultCharset())));
+ jsonMetaDataConfig = FileUtils.readFileToString(file, Charset.defaultCharset());
} catch (final IOException ex) {
throw new FileIOException(ex);
}
diff --git a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java
index e6e81da86c4..63b48149062 100644
--- a/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java
+++ b/proxy/backend/core/src/main/java/org/apache/shardingsphere/proxy/backend/util/YamlDatabaseConfigurationImportExecutor.java
@@ -18,6 +18,10 @@
package org.apache.shardingsphere.proxy.backend.util;
import com.zaxxer.hikari.HikariDataSource;
+import org.apache.shardingsphere.broadcast.api.config.BroadcastRuleConfiguration;
+import org.apache.shardingsphere.broadcast.rule.BroadcastRule;
+import org.apache.shardingsphere.broadcast.yaml.config.YamlBroadcastRuleConfiguration;
+import org.apache.shardingsphere.broadcast.yaml.swapper.YamlBroadcastRuleConfigurationSwapper;
import org.apache.shardingsphere.distsql.handler.exception.DistSQLException;
import org.apache.shardingsphere.distsql.handler.exception.datasource.MissingRequiredDataSourcesException;
import org.apache.shardingsphere.distsql.handler.exception.storageunit.InvalidStorageUnitsException;
@@ -68,6 +72,10 @@ import org.apache.shardingsphere.sharding.api.config.ShardingRuleConfiguration;
import org.apache.shardingsphere.sharding.rule.ShardingRule;
import org.apache.shardingsphere.sharding.yaml.config.YamlShardingRuleConfiguration;
import org.apache.shardingsphere.sharding.yaml.swapper.YamlShardingRuleConfigurationSwapper;
+import org.apache.shardingsphere.single.api.config.SingleRuleConfiguration;
+import org.apache.shardingsphere.single.rule.SingleRule;
+import org.apache.shardingsphere.single.yaml.config.pojo.YamlSingleRuleConfiguration;
+import org.apache.shardingsphere.single.yaml.config.swapper.YamlSingleRuleConfigurationSwapper;
import javax.sql.DataSource;
import java.sql.SQLException;
@@ -192,6 +200,16 @@ public final class YamlDatabaseConfigurationImportExecutor {
MaskRuleConfiguration maskRuleConfig = swapper.swapToObject((YamlMaskRuleConfiguration) each);
ruleConfigsMap.computeIfAbsent(swapper.getOrder(), key -> new LinkedList<>());
ruleConfigsMap.get(swapper.getOrder()).add(maskRuleConfig);
+ } else if (each instanceof YamlBroadcastRuleConfiguration) {
+ YamlBroadcastRuleConfigurationSwapper swapper = new YamlBroadcastRuleConfigurationSwapper();
+ BroadcastRuleConfiguration maskRuleConfig = swapper.swapToObject((YamlBroadcastRuleConfiguration) each);
+ ruleConfigsMap.computeIfAbsent(swapper.getOrder(), key -> new LinkedList<>());
+ ruleConfigsMap.get(swapper.getOrder()).add(maskRuleConfig);
+ } else if (each instanceof YamlSingleRuleConfiguration) {
+ YamlSingleRuleConfigurationSwapper swapper = new YamlSingleRuleConfigurationSwapper();
+ SingleRuleConfiguration maskRuleConfig = swapper.swapToObject((YamlSingleRuleConfiguration) each);
+ ruleConfigsMap.computeIfAbsent(swapper.getOrder(), key -> new LinkedList<>());
+ ruleConfigsMap.get(swapper.getOrder()).add(maskRuleConfig);
}
}
ruleConfigsMap.keySet().stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList())
@@ -216,6 +234,10 @@ public final class YamlDatabaseConfigurationImportExecutor {
ruleConfigs.forEach(each -> addShadowRuleConfiguration((ShadowRuleConfiguration) each, allRuleConfigs, database));
} else if (ruleConfig instanceof MaskRuleConfiguration) {
ruleConfigs.forEach(each -> addMaskRuleConfiguration((MaskRuleConfiguration) each, allRuleConfigs, database));
+ } else if (ruleConfig instanceof BroadcastRuleConfiguration) {
+ ruleConfigs.forEach(each -> addBroadcastRuleConfiguration((BroadcastRuleConfiguration) each, allRuleConfigs, database));
+ } else if (ruleConfig instanceof SingleRuleConfiguration) {
+ ruleConfigs.forEach(each -> addSingleRuleConfiguration((SingleRuleConfiguration) each, allRuleConfigs, database));
}
}
@@ -253,6 +275,16 @@ public final class YamlDatabaseConfigurationImportExecutor {
database.getRuleMetaData().getRules().add(new MaskRule(maskRuleConfig));
}
+ private void addBroadcastRuleConfiguration(final BroadcastRuleConfiguration broadcastRuleConfig, final Collection<RuleConfiguration> allRuleConfigs, final ShardingSphereDatabase database) {
+ allRuleConfigs.add(broadcastRuleConfig);
+ database.getRuleMetaData().getRules().add(new BroadcastRule(broadcastRuleConfig, database.getName(), database.getResourceMetaData().getDataSources()));
+ }
+
+ private void addSingleRuleConfiguration(final SingleRuleConfiguration broadcastRuleConfig, final Collection<RuleConfiguration> allRuleConfigs, final ShardingSphereDatabase database) {
+ allRuleConfigs.add(broadcastRuleConfig);
+ database.getRuleMetaData().getRules().add(new SingleRule(broadcastRuleConfig, database.getName(), database.getResourceMetaData().getDataSources(), database.getRuleMetaData().getRules()));
+ }
+
private void dropDatabase(final String databaseName) {
ProxyContext.getInstance().getContextManager().getInstanceContext().getModeContextManager().dropDatabase(databaseName);
}
diff --git a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java
index 0a6e1de7487..fa2059f2161 100644
--- a/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java
+++ b/proxy/backend/core/src/test/java/org/apache/shardingsphere/proxy/backend/handler/distsql/ral/updatable/ImportMetaDataUpdaterTest.java
@@ -71,7 +71,7 @@ class ImportMetaDataUpdaterTest {
@BeforeEach
void setup() {
- featureMap.put(EMPTY, "/conf/import/empty-metadata.data");
+ featureMap.put(EMPTY, "/conf/import/empty-metadata.json");
}
@Test
diff --git a/proxy/backend/core/src/test/resources/conf/import/empty-metadata.data b/proxy/backend/core/src/test/resources/conf/import/empty-metadata.data
deleted file mode 100644
index e4184a20f34..00000000000
--- a/proxy/backend/core/src/test/resources/conf/import/empty-metadata.data
+++ /dev/null
@@ -1 +0,0 @@
-eyJtZXRhX2RhdGEiOnsiZGF0YWJhc2VzIjp7ImVtcHR5X21ldGFkYXRhIjoiZGF0YWJhc2VOYW1lOiBlbXB0eV9tZXRhZGF0YVxuIn0sInByb3BzIjoiIiwicnVsZXMiOiIifX0=
diff --git a/proxy/backend/core/src/test/resources/conf/import/empty-metadata.json b/proxy/backend/core/src/test/resources/conf/import/empty-metadata.json
new file mode 100644
index 00000000000..ede5beb0d9a
--- /dev/null
+++ b/proxy/backend/core/src/test/resources/conf/import/empty-metadata.json
@@ -0,0 +1 @@
+{"meta_data":{"databases":{"empty_metadata":"databaseName: empty_metadata\n"},"props":"","rules":""}}