You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by su...@apache.org on 2022/12/07 15:18:47 UTC
[shardingsphere] branch master updated: Add SPIDescription (#22731)
This is an automated email from the ASF dual-hosted git repository.
sunnianjun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new d7964f09e1e Add SPIDescription (#22731)
d7964f09e1e is described below
commit d7964f09e1eeea1e035bc27474724dfebd123cf1
Author: Liang Zhang <zh...@apache.org>
AuthorDate: Wed Dec 7 23:18:41 2022 +0800
Add SPIDescription (#22731)
* Add more test cases for RequiredSPIRegistryTest
* Refactor CRC32MatchDataConsistencyCalculateAlgorithm
* Refactor DataConsistencyCalculateAlgorithm
* Refactor DataConsistencyCalculateAlgorithmFixture
* Refactor DataMatchDataConsistencyCalculateAlgorithm
* Refactor MigrationDataConsistencyChecker
* Add SPIDescription
---
.../SPIDescription.java} | 22 ++++++++++------------
.../DataConsistencyCalculateAlgorithm.java | 11 +++++++++--
.../AbstractInventoryIncrementalJobAPIImpl.java | 14 ++++++++++++--
...RC32MatchDataConsistencyCalculateAlgorithm.java | 7 ++-----
...DataMatchDataConsistencyCalculateAlgorithm.java | 7 ++-----
.../MigrationDataConsistencyChecker.java | 6 +++---
.../DataConsistencyCalculateAlgorithmFixture.java | 7 ++-----
7 files changed, 40 insertions(+), 34 deletions(-)
diff --git a/infra/util/src/main/java/org/apache/shardingsphere/infra/util/spi/aware/SPIMetadataAware.java b/infra/util/src/main/java/org/apache/shardingsphere/infra/util/spi/annotation/SPIDescription.java
similarity index 71%
rename from infra/util/src/main/java/org/apache/shardingsphere/infra/util/spi/aware/SPIMetadataAware.java
rename to infra/util/src/main/java/org/apache/shardingsphere/infra/util/spi/annotation/SPIDescription.java
index 3ae8cd37bcd..e54b6939ddc 100644
--- a/infra/util/src/main/java/org/apache/shardingsphere/infra/util/spi/aware/SPIMetadataAware.java
+++ b/infra/util/src/main/java/org/apache/shardingsphere/infra/util/spi/annotation/SPIDescription.java
@@ -15,26 +15,24 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.infra.util.spi.aware;
+package org.apache.shardingsphere.infra.util.spi.annotation;
-import java.util.Collection;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
/**
- * SPI metadata aware.
+ * SPI description.
*/
-public interface SPIMetadataAware {
-
- /**
- * Get supported database types.
- *
- * @return supported database types
- */
- Collection<String> getSupportedDatabaseTypes();
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SPIDescription {
/**
* Get description.
*
* @return description
*/
- String getDescription();
+ String value();
}
diff --git a/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/spi/check/consistency/DataConsistencyCalculateAlgorithm.java b/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/spi/check/consistency/DataConsistencyCalculateAlgorithm.java
index a1f466d6e54..9ffdb8284c3 100644
--- a/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/spi/check/consistency/DataConsistencyCalculateAlgorithm.java
+++ b/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/spi/check/consistency/DataConsistencyCalculateAlgorithm.java
@@ -20,14 +20,14 @@ package org.apache.shardingsphere.data.pipeline.spi.check.consistency;
import org.apache.shardingsphere.data.pipeline.api.check.consistency.DataConsistencyCalculateParameter;
import org.apache.shardingsphere.data.pipeline.api.check.consistency.DataConsistencyCalculatedResult;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithm;
-import org.apache.shardingsphere.infra.util.spi.aware.SPIMetadataAware;
import java.sql.SQLException;
+import java.util.Collection;
/**
* Data consistency calculate algorithm.
*/
-public interface DataConsistencyCalculateAlgorithm extends ShardingSphereAlgorithm, SPIMetadataAware {
+public interface DataConsistencyCalculateAlgorithm extends ShardingSphereAlgorithm {
/**
* Calculate data for consistency check.
@@ -50,4 +50,11 @@ public interface DataConsistencyCalculateAlgorithm extends ShardingSphereAlgorit
* @return canceling or not
*/
boolean isCanceling();
+
+ /**
+ * Get supported database types.
+ *
+ * @return supported database types
+ */
+ Collection<String> getSupportedDatabaseTypes();
}
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/api/impl/AbstractInventoryIncrementalJobAPIImpl.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/api/impl/AbstractInventoryIncrementalJobAPIImpl.java
index 51d34cf1bbc..61776a6b37f 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/api/impl/AbstractInventoryIncrementalJobAPIImpl.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/api/impl/AbstractInventoryIncrementalJobAPIImpl.java
@@ -46,8 +46,10 @@ import org.apache.shardingsphere.data.pipeline.core.task.InventoryTask;
import org.apache.shardingsphere.data.pipeline.spi.check.consistency.DataConsistencyCalculateAlgorithm;
import org.apache.shardingsphere.data.pipeline.spi.check.consistency.DataConsistencyCalculateAlgorithmFactory;
import org.apache.shardingsphere.elasticjob.infra.pojo.JobConfigurationPOJO;
+import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.DatabaseTypeFactory;
import org.apache.shardingsphere.infra.util.exception.ShardingSpherePreconditions;
+import org.apache.shardingsphere.infra.util.spi.annotation.SPIDescription;
import org.apache.shardingsphere.infra.util.yaml.YamlEngine;
import java.util.Collection;
@@ -174,8 +176,16 @@ public abstract class AbstractInventoryIncrementalJobAPIImpl extends AbstractPip
@Override
public Collection<DataConsistencyCheckAlgorithmInfo> listDataConsistencyCheckAlgorithms() {
checkModeConfig();
- return DataConsistencyCalculateAlgorithmFactory.getAllInstances().stream()
- .map(each -> new DataConsistencyCheckAlgorithmInfo(each.getType(), each.getSupportedDatabaseTypes(), each.getDescription())).collect(Collectors.toList());
+ Collection<DataConsistencyCheckAlgorithmInfo> result = new LinkedList<>();
+ for (DataConsistencyCalculateAlgorithm each : DataConsistencyCalculateAlgorithmFactory.getAllInstances()) {
+ SPIDescription spiDescription = each.getClass().getAnnotation(SPIDescription.class);
+ result.add(new DataConsistencyCheckAlgorithmInfo(each.getType(), getSupportedDatabaseTypes(each.getSupportedDatabaseTypes()), null == spiDescription ? "" : spiDescription.value()));
+ }
+ return result;
+ }
+
+ private Collection<String> getSupportedDatabaseTypes(final Collection<String> supportedDatabaseTypes) {
+ return supportedDatabaseTypes.isEmpty() ? DatabaseTypeFactory.getInstances().stream().map(DatabaseType::getType).collect(Collectors.toList()) : supportedDatabaseTypes;
}
@Override
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java
index f4a27a1dc25..5e87b3b7808 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java
@@ -28,6 +28,7 @@ import org.apache.shardingsphere.data.pipeline.core.sqlbuilder.PipelineSQLBuilde
import org.apache.shardingsphere.data.pipeline.spi.sqlbuilder.PipelineSQLBuilder;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import org.apache.shardingsphere.infra.util.exception.ShardingSpherePreconditions;
+import org.apache.shardingsphere.infra.util.spi.annotation.SPIDescription;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -43,6 +44,7 @@ import java.util.stream.Collectors;
/**
* CRC32 match data consistency calculate algorithm.
*/
+@SPIDescription("Match CRC32 of records.")
@Slf4j
public final class CRC32MatchDataConsistencyCalculateAlgorithm extends AbstractDataConsistencyCalculateAlgorithm {
@@ -89,11 +91,6 @@ public final class CRC32MatchDataConsistencyCalculateAlgorithm extends AbstractD
return SUPPORTED_DATABASE_TYPES;
}
- @Override
- public String getDescription() {
- return "Match CRC32 of records.";
- }
-
@RequiredArgsConstructor
@Getter
private static final class CalculatedItem {
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java
index 34d91edb925..236b532d2d0 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java
@@ -34,6 +34,7 @@ import org.apache.shardingsphere.data.pipeline.spi.ingest.dumper.ColumnValueRead
import org.apache.shardingsphere.data.pipeline.spi.sqlbuilder.PipelineSQLBuilder;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.DatabaseTypeFactory;
+import org.apache.shardingsphere.infra.util.spi.annotation.SPIDescription;
import java.math.BigDecimal;
import java.sql.Connection;
@@ -55,6 +56,7 @@ import java.util.stream.Collectors;
/**
* Data match data consistency calculate algorithm.
*/
+@SPIDescription("Match raw data of records.")
@Slf4j
public final class DataMatchDataConsistencyCalculateAlgorithm extends AbstractStreamingDataConsistencyCalculateAlgorithm {
@@ -156,11 +158,6 @@ public final class DataMatchDataConsistencyCalculateAlgorithm extends AbstractSt
return SUPPORTED_DATABASE_TYPES;
}
- @Override
- public String getDescription() {
- return "Match raw data of records.";
- }
-
@RequiredArgsConstructor
@Getter
static final class CalculatedResult implements DataConsistencyCalculatedResult {
diff --git a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/check/consistency/MigrationDataConsistencyChecker.java b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/check/consistency/MigrationDataConsistencyChecker.java
index ea0219ad8ec..314a8143b47 100644
--- a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/check/consistency/MigrationDataConsistencyChecker.java
+++ b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/check/consistency/MigrationDataConsistencyChecker.java
@@ -65,7 +65,7 @@ public final class MigrationDataConsistencyChecker implements PipelineDataConsis
public MigrationDataConsistencyChecker(final MigrationJobConfiguration jobConfig, final InventoryIncrementalProcessContext processContext,
final ConsistencyCheckJobItemProgressContext progressContext) {
this.jobConfig = jobConfig;
- readRateLimitAlgorithm = null != processContext ? processContext.getReadRateLimitAlgorithm() : null;
+ readRateLimitAlgorithm = null == processContext ? null : processContext.getReadRateLimitAlgorithm();
tableNameSchemaNameMapping = new TableNameSchemaNameMapping(
TableNameSchemaNameMapping.convert(jobConfig.getSourceSchemaName(), new HashSet<>(Arrays.asList(jobConfig.getSourceTableName(), jobConfig.getTargetTableName()))));
this.progressContext = progressContext;
@@ -84,8 +84,8 @@ public final class MigrationDataConsistencyChecker implements PipelineDataConsis
progressContext.setRecordsCount(getRecordsCount());
progressContext.getTableNames().add(jobConfig.getSourceTableName());
PipelineTableMetaDataLoader metaDataLoader = new StandardPipelineTableMetaDataLoader(sourceDataSource);
- SingleTableInventoryDataConsistencyChecker singleTableInventoryChecker = new SingleTableInventoryDataConsistencyChecker(jobConfig.getJobId(), sourceDataSource, targetDataSource,
- sourceTable, targetTable, jobConfig.getUniqueKeyColumn(), metaDataLoader, readRateLimitAlgorithm, progressContext);
+ SingleTableInventoryDataConsistencyChecker singleTableInventoryChecker = new SingleTableInventoryDataConsistencyChecker(
+ jobConfig.getJobId(), sourceDataSource, targetDataSource, sourceTable, targetTable, jobConfig.getUniqueKeyColumn(), metaDataLoader, readRateLimitAlgorithm, progressContext);
result.put(sourceTable.getTableName().getOriginal(), singleTableInventoryChecker.check(calculateAlgorithm));
} catch (final SQLException ex) {
throw new SQLWrapperException(ex);
diff --git a/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/fixture/DataConsistencyCalculateAlgorithmFixture.java b/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/fixture/DataConsistencyCalculateAlgorithmFixture.java
index ed3d5466bf0..902d795acdf 100644
--- a/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/fixture/DataConsistencyCalculateAlgorithmFixture.java
+++ b/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/fixture/DataConsistencyCalculateAlgorithmFixture.java
@@ -23,12 +23,14 @@ import org.apache.shardingsphere.data.pipeline.api.check.consistency.DataConsist
import org.apache.shardingsphere.data.pipeline.spi.check.consistency.DataConsistencyCalculateAlgorithm;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.DatabaseTypeFactory;
+import org.apache.shardingsphere.infra.util.spi.annotation.SPIDescription;
import java.util.Collection;
import java.util.Collections;
import java.util.Properties;
import java.util.stream.Collectors;
+@SPIDescription("Fixture description.")
@Getter
public final class DataConsistencyCalculateAlgorithmFixture implements DataConsistencyCalculateAlgorithm {
@@ -58,11 +60,6 @@ public final class DataConsistencyCalculateAlgorithmFixture implements DataConsi
return DatabaseTypeFactory.getInstances().stream().map(DatabaseType::getType).collect(Collectors.toList());
}
- @Override
- public String getDescription() {
- return "Fixture";
- }
-
@Override
public String getType() {
return "FIXTURE";