You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by pa...@apache.org on 2021/12/28 12:12:08 UTC
[shardingsphere] branch master updated: Refactor completionDetectAlgorithm, sourceWritingStopAlgorithm and checkoutLockAlgorithm SPI as general public SPI for common usage (#14381)
This is an automated email from the ASF dual-hosted git repository.
panjuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 68f7364 Refactor completionDetectAlgorithm, sourceWritingStopAlgorithm and checkoutLockAlgorithm SPI as general public SPI for common usage (#14381)
68f7364 is described below
commit 68f7364d7b577ea4cc9ec02e3b8ac34fad5d22b4
Author: Hongsheng Zhong <sa...@126.com>
AuthorDate: Tue Dec 28 20:11:20 2021 +0800
Refactor completionDetectAlgorithm, sourceWritingStopAlgorithm and checkoutLockAlgorithm SPI as general public SPI for common usage (#14381)
---
.../consistency/DataConsistencyCheckerImpl.java | 9 ++-
.../data/pipeline/core/job/FinishedCheckJob.java | 26 +++++----
.../scenario/rulealtered/RuleAlteredContext.java | 25 ++++----
.../RuleAlteredJobProgressDetector.java | 45 ---------------
...a => DefaultMetadataCheckoutLockAlgorithm.java} | 6 +-
...java => DefaultSourceWritingStopAlgorithm.java} | 10 ++--
...dleRuleAlteredJobCompletionDetectAlgorithm.java | 47 ++++++++++++---
...peline.spi.detect.JobCompletionDetectAlgorithm} | 0
...ata.pipeline.spi.lock.RowBasedJobLockAlgorithm} | 2 +-
...ta.pipeline.spi.lock.RuleBasedJobLockAlgorithm} | 2 +-
...=> RuleAlteredJobAlmostCompletedParameter.java} | 17 +++---
.../JobCompletionDetectAlgorithm.java} | 15 ++---
.../JobLock.java} | 11 ++--
.../RowBasedJobLockAlgorithm.java} | 22 +-------
.../lock/RuleBasedJobLockAlgorithm.java} | 20 ++-----
...va => FixtureJobCompletionDetectAlgorithm.java} | 7 +--
...uleAlteredJobCompletionDetectAlgorithmTest.java | 66 +++++++++++++++++-----
...peline.spi.detect.JobCompletionDetectAlgorithm} | 2 +-
18 files changed, 170 insertions(+), 162 deletions(-)
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java
index 3faca19..3bc358a 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java
@@ -189,13 +189,18 @@ public final class DataConsistencyCheckerImpl implements DataConsistencyChecker
try (PipelineDataSourceWrapper dataSource = dataSourceFactory.newInstance(dataSourceConfig)) {
Map<String, TableMetaData> result = new LinkedHashMap<>();
for (String each : tableNames) {
- Optional<TableMetaData> tableMetaDataOptional = TableMetaDataLoader.load(dataSource, each, dataSourceConfig.getDatabaseType());
+ Optional<TableMetaData> tableMetaDataOptional;
+ try {
+ tableMetaDataOptional = TableMetaDataLoader.load(dataSource, each, dataSourceConfig.getDatabaseType());
+ } catch (final SQLException ex) {
+ throw new PipelineDataConsistencyCheckFailedException(String.format("get columns failed for table '%s'", each), ex);
+ }
TableMetaData tableMetaData = tableMetaDataOptional.orElseThrow(() -> new PipelineDataConsistencyCheckFailedException(String.format("get metadata failed for table '%s'", each)));
result.put(each, tableMetaData);
}
return result;
} catch (final SQLException ex) {
- throw new PipelineDataConsistencyCheckFailedException("get table columns failed", ex);
+ throw new PipelineDataConsistencyCheckFailedException("create data source failed", ex);
}
}
}
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/job/FinishedCheckJob.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/job/FinishedCheckJob.java
index 3375c5d..bc771db 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/job/FinishedCheckJob.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/job/FinishedCheckJob.java
@@ -22,12 +22,12 @@ import org.apache.shardingsphere.data.pipeline.api.PipelineJobAPI;
import org.apache.shardingsphere.data.pipeline.api.PipelineJobAPIFactory;
import org.apache.shardingsphere.data.pipeline.api.check.consistency.DataConsistencyCheckResult;
import org.apache.shardingsphere.data.pipeline.api.config.rulealtered.JobConfiguration;
+import org.apache.shardingsphere.data.pipeline.api.detect.RuleAlteredJobAlmostCompletedParameter;
import org.apache.shardingsphere.data.pipeline.api.pojo.JobInfo;
import org.apache.shardingsphere.data.pipeline.scenario.rulealtered.RuleAlteredContext;
-import org.apache.shardingsphere.data.pipeline.scenario.rulealtered.RuleAlteredJobProgressDetector;
import org.apache.shardingsphere.data.pipeline.scenario.rulealtered.RuleAlteredJobWorker;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredCheckoutLockAlgorithm;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredSourceWritingStopAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.lock.RuleBasedJobLockAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.lock.RowBasedJobLockAlgorithm;
import org.apache.shardingsphere.elasticjob.api.ShardingContext;
import org.apache.shardingsphere.elasticjob.simple.job.SimpleJob;
import org.apache.shardingsphere.infra.yaml.engine.YamlEngine;
@@ -52,18 +52,20 @@ public final class FinishedCheckJob implements SimpleJob {
// TODO refactor: dispatch to different job types
JobConfiguration jobConfig = YamlEngine.unmarshal(jobInfo.getJobParameter(), JobConfiguration.class, true);
RuleAlteredContext ruleAlteredContext = RuleAlteredJobWorker.createRuleAlteredContext(jobConfig);
- if (!RuleAlteredJobProgressDetector.almostFinished(pipelineJobAPI.getProgress(jobId), jobConfig.getHandleConfig(), ruleAlteredContext)) {
+ if (null == ruleAlteredContext.getCompletionDetectAlgorithm()) {
+ log.info("completionDetector not configured, auto switch will not be enabled. You could query job progress and switch config manually with DistSQL.");
continue;
}
- log.info("scaling job {} almost finished.", jobId);
- if (null == ruleAlteredContext.getCompletionDetectAlgorithm()) {
- log.info("completionDetector not configured, auto switch will not be enabled. You could query migration progress and switch manually with DistSQL.");
+ RuleAlteredJobAlmostCompletedParameter parameter = new RuleAlteredJobAlmostCompletedParameter(jobInfo.getShardingTotalCount(), pipelineJobAPI.getProgress(jobId).values());
+ if (!ruleAlteredContext.getCompletionDetectAlgorithm().isAlmostCompleted(parameter)) {
+ continue;
}
- RuleAlteredSourceWritingStopAlgorithm sourceWritingStopAlgorithm = ruleAlteredContext.getSourceWritingStopAlgorithm();
+ log.info("scaling job {} almost finished.", jobId);
+ RowBasedJobLockAlgorithm sourceWritingStopAlgorithm = ruleAlteredContext.getSourceWritingStopAlgorithm();
String schemaName = jobConfig.getWorkflowConfig().getSchemaName();
try {
if (null != sourceWritingStopAlgorithm) {
- sourceWritingStopAlgorithm.stopSourceWriting(schemaName, jobId + "");
+ sourceWritingStopAlgorithm.lock(schemaName, jobId + "");
}
if (!pipelineJobAPI.isDataConsistencyCheckNeeded(jobId)) {
log.info("dataConsistencyCheckAlgorithm is not configured, data consistency check is ignored.");
@@ -74,11 +76,11 @@ public final class FinishedCheckJob implements SimpleJob {
log.error("data consistency check failed, job {}", jobId);
continue;
}
- RuleAlteredCheckoutLockAlgorithm checkoutLockAlgorithm = ruleAlteredContext.getCheckoutLockAlgorithm();
+ RuleBasedJobLockAlgorithm checkoutLockAlgorithm = ruleAlteredContext.getCheckoutLockAlgorithm();
switchClusterConfiguration(schemaName, jobId, checkoutLockAlgorithm);
} finally {
if (null != sourceWritingStopAlgorithm) {
- sourceWritingStopAlgorithm.resumeSourceWriting(schemaName, jobId + "");
+ sourceWritingStopAlgorithm.releaseLock(schemaName, jobId + "");
}
}
log.info("job {} finished", jobId);
@@ -95,7 +97,7 @@ public final class FinishedCheckJob implements SimpleJob {
return pipelineJobAPI.aggregateDataConsistencyCheckResults(jobId, checkResultMap);
}
- private void switchClusterConfiguration(final String schemaName, final long jobId, final RuleAlteredCheckoutLockAlgorithm checkoutLockAlgorithm) {
+ private void switchClusterConfiguration(final String schemaName, final long jobId, final RuleBasedJobLockAlgorithm checkoutLockAlgorithm) {
try {
if (null != checkoutLockAlgorithm) {
checkoutLockAlgorithm.lock(schemaName, jobId + "");
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredContext.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredContext.java
index b73fd5c..5dde8a9 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredContext.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredContext.java
@@ -19,12 +19,13 @@ package org.apache.shardingsphere.data.pipeline.scenario.rulealtered;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
+import org.apache.shardingsphere.data.pipeline.api.detect.RuleAlteredJobAlmostCompletedParameter;
import org.apache.shardingsphere.data.pipeline.core.execute.ExecuteEngine;
import org.apache.shardingsphere.data.pipeline.spi.check.consistency.DataConsistencyCheckAlgorithm;
import org.apache.shardingsphere.data.pipeline.spi.ratelimit.JobRateLimitAlgorithm;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredCheckoutLockAlgorithm;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredSourceWritingStopAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.lock.RuleBasedJobLockAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.lock.RowBasedJobLockAlgorithm;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmConfiguration;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmFactory;
import org.apache.shardingsphere.infra.config.mode.ModeConfiguration;
@@ -41,10 +42,10 @@ public final class RuleAlteredContext {
static {
ShardingSphereServiceLoader.register(JobRateLimitAlgorithm.class);
- ShardingSphereServiceLoader.register(RuleAlteredJobCompletionDetectAlgorithm.class);
- ShardingSphereServiceLoader.register(RuleAlteredSourceWritingStopAlgorithm.class);
+ ShardingSphereServiceLoader.register(JobCompletionDetectAlgorithm.class);
+ ShardingSphereServiceLoader.register(RowBasedJobLockAlgorithm.class);
ShardingSphereServiceLoader.register(DataConsistencyCheckAlgorithm.class);
- ShardingSphereServiceLoader.register(RuleAlteredCheckoutLockAlgorithm.class);
+ ShardingSphereServiceLoader.register(RuleBasedJobLockAlgorithm.class);
}
private static volatile ModeConfiguration modeConfig;
@@ -53,13 +54,13 @@ public final class RuleAlteredContext {
private final JobRateLimitAlgorithm rateLimitAlgorithm;
- private final RuleAlteredJobCompletionDetectAlgorithm completionDetectAlgorithm;
+ private final JobCompletionDetectAlgorithm<RuleAlteredJobAlmostCompletedParameter> completionDetectAlgorithm;
- private final RuleAlteredSourceWritingStopAlgorithm sourceWritingStopAlgorithm;
+ private final RowBasedJobLockAlgorithm sourceWritingStopAlgorithm;
private final DataConsistencyCheckAlgorithm dataConsistencyCheckAlgorithm;
- private final RuleAlteredCheckoutLockAlgorithm checkoutLockAlgorithm;
+ private final RuleBasedJobLockAlgorithm checkoutLockAlgorithm;
private final ExecuteEngine inventoryDumperExecuteEngine;
@@ -77,13 +78,13 @@ public final class RuleAlteredContext {
}
ShardingSphereAlgorithmConfiguration completionDetector = onRuleAlteredActionConfig.getCompletionDetector();
if (null != completionDetector) {
- completionDetectAlgorithm = ShardingSphereAlgorithmFactory.createAlgorithm(completionDetector, RuleAlteredJobCompletionDetectAlgorithm.class);
+ completionDetectAlgorithm = ShardingSphereAlgorithmFactory.createAlgorithm(completionDetector, JobCompletionDetectAlgorithm.class);
} else {
completionDetectAlgorithm = null;
}
ShardingSphereAlgorithmConfiguration sourceWritingStopper = onRuleAlteredActionConfig.getSourceWritingStopper();
if (null != sourceWritingStopper) {
- sourceWritingStopAlgorithm = ShardingSphereAlgorithmFactory.createAlgorithm(sourceWritingStopper, RuleAlteredSourceWritingStopAlgorithm.class);
+ sourceWritingStopAlgorithm = ShardingSphereAlgorithmFactory.createAlgorithm(sourceWritingStopper, RowBasedJobLockAlgorithm.class);
} else {
sourceWritingStopAlgorithm = null;
}
@@ -95,7 +96,7 @@ public final class RuleAlteredContext {
}
ShardingSphereAlgorithmConfiguration checkoutLocker = onRuleAlteredActionConfig.getCheckoutLocker();
if (null != checkoutLocker) {
- checkoutLockAlgorithm = ShardingSphereAlgorithmFactory.createAlgorithm(checkoutLocker, RuleAlteredCheckoutLockAlgorithm.class);
+ checkoutLockAlgorithm = ShardingSphereAlgorithmFactory.createAlgorithm(checkoutLocker, RuleBasedJobLockAlgorithm.class);
} else {
checkoutLockAlgorithm = null;
}
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobProgressDetector.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobProgressDetector.java
index d7a4aed..7d777ae 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobProgressDetector.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/RuleAlteredJobProgressDetector.java
@@ -20,18 +20,10 @@ package org.apache.shardingsphere.data.pipeline.scenario.rulealtered;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.shardingsphere.data.pipeline.api.config.rulealtered.HandleConfiguration;
-import org.apache.shardingsphere.data.pipeline.api.detect.AllIncrementalTasksAlmostFinishedParameter;
import org.apache.shardingsphere.data.pipeline.api.ingest.position.FinishedPosition;
-import org.apache.shardingsphere.data.pipeline.api.job.progress.JobProgress;
import org.apache.shardingsphere.data.pipeline.core.task.InventoryTask;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm;
import java.util.Collection;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
/**
* Rule altered job progress detector.
@@ -41,37 +33,6 @@ import java.util.stream.Collectors;
public final class RuleAlteredJobProgressDetector {
/**
- * All inventory tasks is finished and all incremental tasks is almost finished.
- *
- * @param jobProgressMap job progress map
- * @param handleConfig handle configuration
- * @param ruleAlteredContext rule altered context
- * @return almost finished or not
- */
- public static boolean almostFinished(final Map<Integer, JobProgress> jobProgressMap, final HandleConfiguration handleConfig, final RuleAlteredContext ruleAlteredContext) {
- return isProgressCompleted(jobProgressMap, handleConfig)
- && allInventoryTasksFinished(jobProgressMap)
- && allIncrementalTasksAlmostFinished(jobProgressMap, ruleAlteredContext.getCompletionDetectAlgorithm());
- }
-
- private static boolean isProgressCompleted(final Map<Integer, JobProgress> jobProgressMap, final HandleConfiguration handleConfig) {
- return handleConfig.getJobShardingCount() == jobProgressMap.size()
- && jobProgressMap.values().stream().allMatch(Objects::nonNull);
- }
-
- private static boolean allIncrementalTasksAlmostFinished(final Map<Integer, JobProgress> jobProgressMap, final RuleAlteredJobCompletionDetectAlgorithm completionDetectAlgorithm) {
- long currentTimeMillis = System.currentTimeMillis();
- Collection<Long> incrementalTaskIdleMinutes = jobProgressMap.values().stream().flatMap(each -> each.getIncrementalTaskProgressMap().values().stream())
- .map(each -> {
- long latestActiveTimeMillis = each.getIncrementalTaskDelay().getLatestActiveTimeMillis();
- return latestActiveTimeMillis > 0 ? TimeUnit.MILLISECONDS.toMinutes(currentTimeMillis - latestActiveTimeMillis) : 0;
- })
- .collect(Collectors.toList());
- AllIncrementalTasksAlmostFinishedParameter parameter = AllIncrementalTasksAlmostFinishedParameter.builder().incrementalTaskIdleMinutes(incrementalTaskIdleMinutes).build();
- return completionDetectAlgorithm.allIncrementalTasksAlmostFinished(parameter);
- }
-
- /**
* All inventory tasks is finished.
*
* @param inventoryTasks to check inventory tasks
@@ -83,10 +44,4 @@ public final class RuleAlteredJobProgressDetector {
}
return inventoryTasks.stream().allMatch(each -> each.getProgress().getPosition() instanceof FinishedPosition);
}
-
- private static boolean allInventoryTasksFinished(final Map<Integer, JobProgress> jobProgress) {
- return jobProgress.values().stream()
- .flatMap(each -> each.getInventoryTaskProgressMap().values().stream())
- .allMatch(each -> each.getPosition() instanceof FinishedPosition);
- }
}
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultRuleAlteredCheckoutLockAlgorithm.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultMetadataCheckoutLockAlgorithm.java
similarity index 83%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultRuleAlteredCheckoutLockAlgorithm.java
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultMetadataCheckoutLockAlgorithm.java
index db93ae5..be45f29 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultRuleAlteredCheckoutLockAlgorithm.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultMetadataCheckoutLockAlgorithm.java
@@ -18,13 +18,13 @@
package org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi;
import lombok.ToString;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredCheckoutLockAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.lock.RuleBasedJobLockAlgorithm;
/**
- * Default rule altered checkout lock algorithm.
+ * Default metadata checkout lock algorithm.
*/
@ToString
-public final class DefaultRuleAlteredCheckoutLockAlgorithm implements RuleAlteredCheckoutLockAlgorithm {
+public final class DefaultMetadataCheckoutLockAlgorithm implements RuleBasedJobLockAlgorithm {
@Override
public void init() {
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultRuleAlteredSourceWritingStopAlgorithm.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultSourceWritingStopAlgorithm.java
similarity index 72%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultRuleAlteredSourceWritingStopAlgorithm.java
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultSourceWritingStopAlgorithm.java
index 2215088..70a7f0d 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultRuleAlteredSourceWritingStopAlgorithm.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/DefaultSourceWritingStopAlgorithm.java
@@ -18,13 +18,13 @@
package org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi;
import lombok.ToString;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredSourceWritingStopAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.lock.RowBasedJobLockAlgorithm;
/**
- * Default rule altered source writing stop algorithm.
+ * Default source writing stop algorithm.
*/
@ToString
-public final class DefaultRuleAlteredSourceWritingStopAlgorithm implements RuleAlteredSourceWritingStopAlgorithm {
+public final class DefaultSourceWritingStopAlgorithm implements RowBasedJobLockAlgorithm {
@Override
public void init() {
@@ -32,11 +32,11 @@ public final class DefaultRuleAlteredSourceWritingStopAlgorithm implements RuleA
// TODO impl default sourceWritingStopAlgorithm
@Override
- public void stopSourceWriting(final String schemaName, final String jobId) {
+ public void lock(final String schemaName, final String jobId) {
}
@Override
- public void resumeSourceWriting(final String schemaName, final String jobId) {
+ public void releaseLock(final String schemaName, final String jobId) {
}
@Override
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/IdleRuleAlteredJobCompletionDetectAlgorithm.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/IdleRuleAlteredJobCompletionDetectAlgorithm.java
index f06ee3f..c6af445 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/IdleRuleAlteredJobCompletionDetectAlgorithm.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/spi/IdleRuleAlteredJobCompletionDetectAlgorithm.java
@@ -18,21 +18,28 @@
package org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi;
import com.google.common.base.Preconditions;
-import org.apache.shardingsphere.data.pipeline.api.detect.AllIncrementalTasksAlmostFinishedParameter;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm;
+import lombok.Getter;
+import org.apache.shardingsphere.data.pipeline.api.detect.RuleAlteredJobAlmostCompletedParameter;
+import org.apache.shardingsphere.data.pipeline.api.ingest.position.FinishedPosition;
+import org.apache.shardingsphere.data.pipeline.api.job.progress.JobProgress;
+import org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm;
import java.util.Collection;
+import java.util.Objects;
import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
/**
* Idle rule altered job completion detect algorithm.
*/
-public final class IdleRuleAlteredJobCompletionDetectAlgorithm implements RuleAlteredJobCompletionDetectAlgorithm {
+public final class IdleRuleAlteredJobCompletionDetectAlgorithm implements JobCompletionDetectAlgorithm<RuleAlteredJobAlmostCompletedParameter> {
public static final String IDLE_THRESHOLD_KEY = "incremental-task-idle-minute-threshold";
private Properties props = new Properties();
+ @Getter
private long incrementalTaskIdleMinuteThreshold = 30;
@Override
@@ -58,12 +65,38 @@ public final class IdleRuleAlteredJobCompletionDetectAlgorithm implements RuleAl
}
@Override
- public boolean allIncrementalTasksAlmostFinished(final AllIncrementalTasksAlmostFinishedParameter parameter) {
- Collection<Long> incrementalTaskIdleMinutes = parameter.getIncrementalTaskIdleMinutes();
- if (null == incrementalTaskIdleMinutes || incrementalTaskIdleMinutes.isEmpty()) {
+ public boolean isAlmostCompleted(final RuleAlteredJobAlmostCompletedParameter parameter) {
+ int jobShardingCount = parameter.getJobShardingCount();
+ Collection<JobProgress> jobProgresses = parameter.getJobProgresses();
+ if (!isAllProgressesFilled(jobShardingCount, jobProgresses)) {
return false;
}
- return incrementalTaskIdleMinutes.stream().allMatch(idleMinute -> idleMinute >= incrementalTaskIdleMinuteThreshold);
+ if (!isAllInventoryTasksCompleted(jobProgresses)) {
+ return false;
+ }
+ Collection<Long> incrementalTasksIdleMinutes = getIncrementalTasksIdleMinutes(jobProgresses);
+ return incrementalTasksIdleMinutes.stream().allMatch(idleMinute -> idleMinute >= incrementalTaskIdleMinuteThreshold);
+ }
+
+ private static boolean isAllProgressesFilled(final int jobShardingCount, final Collection<JobProgress> jobProgresses) {
+ return jobShardingCount == jobProgresses.size()
+ && jobProgresses.stream().allMatch(Objects::nonNull);
+ }
+
+ private static boolean isAllInventoryTasksCompleted(final Collection<JobProgress> jobProgresses) {
+ return jobProgresses.stream()
+ .flatMap(each -> each.getInventoryTaskProgressMap().values().stream())
+ .allMatch(each -> each.getPosition() instanceof FinishedPosition);
+ }
+
+ private static Collection<Long> getIncrementalTasksIdleMinutes(final Collection<JobProgress> jobProgresses) {
+ long currentTimeMillis = System.currentTimeMillis();
+ return jobProgresses.stream().flatMap(each -> each.getIncrementalTaskProgressMap().values().stream())
+ .map(each -> {
+ long latestActiveTimeMillis = each.getIncrementalTaskDelay().getLatestActiveTimeMillis();
+ return latestActiveTimeMillis > 0 ? TimeUnit.MILLISECONDS.toMinutes(currentTimeMillis - latestActiveTimeMillis) : 0;
+ })
+ .collect(Collectors.toList());
}
@Override
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm
similarity index 100%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredCheckoutLockAlgorithm b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.lock.RowBasedJobLockAlgorithm
similarity index 95%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredCheckoutLockAlgorithm
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.lock.RowBasedJobLockAlgorithm
index 62c4c28..725247f 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredCheckoutLockAlgorithm
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.lock.RowBasedJobLockAlgorithm
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi.DefaultRuleAlteredCheckoutLockAlgorithm
+org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi.DefaultSourceWritingStopAlgorithm
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredSourceWritingStopAlgorithm b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.lock.RuleBasedJobLockAlgorithm
similarity index 94%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredSourceWritingStopAlgorithm
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.lock.RuleBasedJobLockAlgorithm
index fb0f37d..3850001 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredSourceWritingStopAlgorithm
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.lock.RuleBasedJobLockAlgorithm
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi.DefaultRuleAlteredSourceWritingStopAlgorithm
+org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi.DefaultMetadataCheckoutLockAlgorithm
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/AllIncrementalTasksAlmostFinishedParameter.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/RuleAlteredJobAlmostCompletedParameter.java
similarity index 71%
copy from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/AllIncrementalTasksAlmostFinishedParameter.java
copy to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/RuleAlteredJobAlmostCompletedParameter.java
index ddff3c3..58d32a5 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/AllIncrementalTasksAlmostFinishedParameter.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/RuleAlteredJobAlmostCompletedParameter.java
@@ -17,21 +17,24 @@
package org.apache.shardingsphere.data.pipeline.api.detect;
-import lombok.Builder;
import lombok.Getter;
-import lombok.Setter;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
import lombok.ToString;
+import org.apache.shardingsphere.data.pipeline.api.job.progress.JobProgress;
import java.util.Collection;
/**
- * All incremental tasks almost finished parameter.
+ * Rule altered job almost completed parameter.
*/
+@RequiredArgsConstructor
@Getter
-@Setter
-@Builder
@ToString
-public final class AllIncrementalTasksAlmostFinishedParameter {
+public final class RuleAlteredJobAlmostCompletedParameter {
- private Collection<Long> incrementalTaskIdleMinutes;
+ private final int jobShardingCount;
+
+ @NonNull
+ private final Collection<JobProgress> jobProgresses;
}
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredJobCompletionDetectAlgorithm.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/detect/JobCompletionDetectAlgorithm.java
similarity index 66%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredJobCompletionDetectAlgorithm.java
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/detect/JobCompletionDetectAlgorithm.java
index 164035c..86dd8ca 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredJobCompletionDetectAlgorithm.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/detect/JobCompletionDetectAlgorithm.java
@@ -15,22 +15,23 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.data.pipeline.spi.rulealtered;
+package org.apache.shardingsphere.data.pipeline.spi.detect;
-import org.apache.shardingsphere.data.pipeline.api.detect.AllIncrementalTasksAlmostFinishedParameter;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithm;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmPostProcessor;
/**
- * Rule altered job completion detect algorithm for SPI.
+ * Job completion detect algorithm, SPI.
+ *
+ * @param <T> completion detect parameter type
*/
-public interface RuleAlteredJobCompletionDetectAlgorithm extends ShardingSphereAlgorithm, ShardingSphereAlgorithmPostProcessor {
+public interface JobCompletionDetectAlgorithm<T> extends ShardingSphereAlgorithm, ShardingSphereAlgorithmPostProcessor {
/**
- * All incremental tasks is almost finished.
+ * Whether job is almost completed.
*
* @param parameter parameter
- * @return Almost finished or not
+ * @return almost completed or not
*/
- boolean allIncrementalTasksAlmostFinished(AllIncrementalTasksAlmostFinishedParameter parameter);
+ boolean isAlmostCompleted(T parameter);
}
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredCheckoutLockAlgorithm.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/JobLock.java
similarity index 70%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredCheckoutLockAlgorithm.java
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/JobLock.java
index 805059d..2487512 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredCheckoutLockAlgorithm.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/JobLock.java
@@ -15,18 +15,15 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.data.pipeline.spi.rulealtered;
-
-import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithm;
-import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmPostProcessor;
+package org.apache.shardingsphere.data.pipeline.spi.lock;
/**
- * Rule altered checkout lock algorithm, SPI.
+ * Job lock.
*/
-public interface RuleAlteredCheckoutLockAlgorithm extends ShardingSphereAlgorithm, ShardingSphereAlgorithmPostProcessor {
+public interface JobLock {
/**
- * Lock to protect checkout (switch configuration).
+ * Lock.
*
* @param schemaName schema name
* @param jobId job id
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredSourceWritingStopAlgorithm.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/RowBasedJobLockAlgorithm.java
similarity index 61%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredSourceWritingStopAlgorithm.java
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/RowBasedJobLockAlgorithm.java
index ff753b8..fefeac3 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/rulealtered/RuleAlteredSourceWritingStopAlgorithm.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/RowBasedJobLockAlgorithm.java
@@ -15,29 +15,13 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.data.pipeline.spi.rulealtered;
+package org.apache.shardingsphere.data.pipeline.spi.lock;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithm;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmPostProcessor;
/**
- * Rule altered source writing stop algorithm, SPI.
+ * Row based job lock algorithm, SPI.
*/
-public interface RuleAlteredSourceWritingStopAlgorithm extends ShardingSphereAlgorithm, ShardingSphereAlgorithmPostProcessor {
-
- /**
- * Stop source writing.
- *
- * @param schemaName schema name
- * @param jobId job id
- */
- void stopSourceWriting(String schemaName, String jobId);
-
- /**
- * Resume source writing.
- *
- * @param schemaName schema name
- * @param jobId job id
- */
- void resumeSourceWriting(String schemaName, String jobId);
+public interface RowBasedJobLockAlgorithm extends JobLock, ShardingSphereAlgorithm, ShardingSphereAlgorithmPostProcessor {
}
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/AllIncrementalTasksAlmostFinishedParameter.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/RuleBasedJobLockAlgorithm.java
similarity index 67%
rename from shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/AllIncrementalTasksAlmostFinishedParameter.java
rename to shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/RuleBasedJobLockAlgorithm.java
index ddff3c3..2926578 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/api/detect/AllIncrementalTasksAlmostFinishedParameter.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-spi/src/main/java/org/apache/shardingsphere/data/pipeline/spi/lock/RuleBasedJobLockAlgorithm.java
@@ -15,23 +15,13 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.data.pipeline.api.detect;
+package org.apache.shardingsphere.data.pipeline.spi.lock;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.ToString;
-
-import java.util.Collection;
+import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithm;
+import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmPostProcessor;
/**
- * All incremental tasks almost finished parameter.
+ * Rule based job lock algorithm, SPI.
*/
-@Getter
-@Setter
-@Builder
-@ToString
-public final class AllIncrementalTasksAlmostFinishedParameter {
-
- private Collection<Long> incrementalTaskIdleMinutes;
+public interface RuleBasedJobLockAlgorithm extends JobLock, ShardingSphereAlgorithm, ShardingSphereAlgorithmPostProcessor {
}
diff --git a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/fixture/FixtureRuleAlteredJobCompletionDetectAlgorithm.java b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/fixture/FixtureJobCompletionDetectAlgorithm.java
similarity index 70%
rename from shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/fixture/FixtureRuleAlteredJobCompletionDetectAlgorithm.java
rename to shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/fixture/FixtureJobCompletionDetectAlgorithm.java
index 33eeb9f..140ffeb 100644
--- a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/fixture/FixtureRuleAlteredJobCompletionDetectAlgorithm.java
+++ b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/fixture/FixtureJobCompletionDetectAlgorithm.java
@@ -17,17 +17,16 @@
package org.apache.shardingsphere.data.pipeline.core.fixture;
-import org.apache.shardingsphere.data.pipeline.api.detect.AllIncrementalTasksAlmostFinishedParameter;
-import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm;
+import org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm;
-public final class FixtureRuleAlteredJobCompletionDetectAlgorithm implements RuleAlteredJobCompletionDetectAlgorithm {
+public final class FixtureJobCompletionDetectAlgorithm implements JobCompletionDetectAlgorithm<Object> {
@Override
public void init() {
}
@Override
- public boolean allIncrementalTasksAlmostFinished(final AllIncrementalTasksAlmostFinishedParameter parameter) {
+ public boolean isAlmostCompleted(final Object parameter) {
return true;
}
diff --git a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/IdleRuleAlteredJobCompletionDetectAlgorithmTest.java b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/IdleRuleAlteredJobCompletionDetectAlgorithmTest.java
index 01064dc..ce34975 100644
--- a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/IdleRuleAlteredJobCompletionDetectAlgorithmTest.java
+++ b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/scenario/rulealtered/IdleRuleAlteredJobCompletionDetectAlgorithmTest.java
@@ -17,7 +17,12 @@
package org.apache.shardingsphere.data.pipeline.scenario.rulealtered;
-import org.apache.shardingsphere.data.pipeline.api.detect.AllIncrementalTasksAlmostFinishedParameter;
+import org.apache.shardingsphere.data.pipeline.api.detect.RuleAlteredJobAlmostCompletedParameter;
+import org.apache.shardingsphere.data.pipeline.api.ingest.position.FinishedPosition;
+import org.apache.shardingsphere.data.pipeline.api.ingest.position.PlaceholderPosition;
+import org.apache.shardingsphere.data.pipeline.api.job.progress.JobProgress;
+import org.apache.shardingsphere.data.pipeline.api.task.progress.IncrementalTaskProgress;
+import org.apache.shardingsphere.data.pipeline.api.task.progress.InventoryTaskProgress;
import org.apache.shardingsphere.data.pipeline.core.util.ReflectionUtil;
import org.apache.shardingsphere.data.pipeline.scenario.rulealtered.spi.IdleRuleAlteredJobCompletionDetectAlgorithm;
import org.junit.Before;
@@ -27,9 +32,13 @@ import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.junit.MockitoJUnitRunner;
-import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertFalse;
@@ -84,26 +93,55 @@ public final class IdleRuleAlteredJobCompletionDetectAlgorithmTest {
}
@Test
- public void assertFalseOnNullIncrementalTasks() {
- AllIncrementalTasksAlmostFinishedParameter parameter = AllIncrementalTasksAlmostFinishedParameter.builder().build();
- assertFalse(detectAlgorithm.allIncrementalTasksAlmostFinished(parameter));
+ public void assertFalseOnFewJobProgresses() {
+ int jobShardingCount = 2;
+ Collection<JobProgress> jobProgresses = Collections.singleton(new JobProgress());
+ RuleAlteredJobAlmostCompletedParameter parameter = new RuleAlteredJobAlmostCompletedParameter(jobShardingCount, jobProgresses);
+ assertFalse(detectAlgorithm.isAlmostCompleted(parameter));
}
@Test
- public void assertFalseOnEmptyIncrementalTasks() {
- AllIncrementalTasksAlmostFinishedParameter parameter = AllIncrementalTasksAlmostFinishedParameter.builder().incrementalTaskIdleMinutes(Collections.emptyList()).build();
- assertFalse(detectAlgorithm.allIncrementalTasksAlmostFinished(parameter));
+ public void assertFalseOnUnFinishedPosition() {
+ int jobShardingCount = 1;
+ JobProgress jobProgress = new JobProgress();
+ Map<String, InventoryTaskProgress> inventoryTaskProgressMap = new LinkedHashMap<>();
+ jobProgress.setInventoryTaskProgressMap(inventoryTaskProgressMap);
+ inventoryTaskProgressMap.put("ds_0", new InventoryTaskProgress(new PlaceholderPosition()));
+ Collection<JobProgress> jobProgresses = Collections.singleton(jobProgress);
+ RuleAlteredJobAlmostCompletedParameter parameter = new RuleAlteredJobAlmostCompletedParameter(jobShardingCount, jobProgresses);
+ assertFalse(detectAlgorithm.isAlmostCompleted(parameter));
}
@Test
- public void assertFalseOnFewPendingIncrementalTasks() {
- AllIncrementalTasksAlmostFinishedParameter parameter = AllIncrementalTasksAlmostFinishedParameter.builder().incrementalTaskIdleMinutes(Arrays.asList(10L, 50L)).build();
- assertFalse(detectAlgorithm.allIncrementalTasksAlmostFinished(parameter));
+ public void assertTrueWhenIdleMinutesNotReach() {
+ int jobShardingCount = 1;
+ long latestActiveTimeMillis = System.currentTimeMillis() - ThreadLocalRandom.current().nextLong(1, detectAlgorithm.getIncrementalTaskIdleMinuteThreshold());
+ JobProgress jobProgress = createJobProgress(latestActiveTimeMillis);
+ Collection<JobProgress> jobProgresses = Collections.singleton(jobProgress);
+ RuleAlteredJobAlmostCompletedParameter parameter = new RuleAlteredJobAlmostCompletedParameter(jobShardingCount, jobProgresses);
+ assertFalse(detectAlgorithm.isAlmostCompleted(parameter));
+ }
+
+ private JobProgress createJobProgress(final long latestActiveTimeMillis) {
+ JobProgress result = new JobProgress();
+ Map<String, InventoryTaskProgress> inventoryTaskProgressMap = new LinkedHashMap<>();
+ result.setInventoryTaskProgressMap(inventoryTaskProgressMap);
+ inventoryTaskProgressMap.put("ds_0", new InventoryTaskProgress(new FinishedPosition()));
+ Map<String, IncrementalTaskProgress> incrementalTaskProgressMap = new LinkedHashMap<>();
+ result.setIncrementalTaskProgressMap(incrementalTaskProgressMap);
+ IncrementalTaskProgress incrementalTaskProgress = new IncrementalTaskProgress();
+ incrementalTaskProgress.getIncrementalTaskDelay().setLatestActiveTimeMillis(latestActiveTimeMillis);
+ incrementalTaskProgressMap.put("ds_0", incrementalTaskProgress);
+ return result;
}
@Test
- public void assertTrueWhenAllIncrementalTasksAlmostFinished() {
- AllIncrementalTasksAlmostFinishedParameter parameter = AllIncrementalTasksAlmostFinishedParameter.builder().incrementalTaskIdleMinutes(Arrays.asList(60L, 50L, 30L)).build();
- assertTrue(detectAlgorithm.allIncrementalTasksAlmostFinished(parameter));
+ public void assertTrueWhenJobAlmostCompleted() {
+ int jobShardingCount = 1;
+ long latestActiveTimeMillis = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(detectAlgorithm.getIncrementalTaskIdleMinuteThreshold() + 5);
+ JobProgress jobProgress = createJobProgress(latestActiveTimeMillis);
+ Collection<JobProgress> jobProgresses = Collections.singleton(jobProgress);
+ RuleAlteredJobAlmostCompletedParameter parameter = new RuleAlteredJobAlmostCompletedParameter(jobShardingCount, jobProgresses);
+ assertTrue(detectAlgorithm.isAlmostCompleted(parameter));
}
}
diff --git a/shardingsphere-test/shardingsphere-pipeline-test/src/test/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm b/shardingsphere-test/shardingsphere-pipeline-test/src/test/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm
similarity index 88%
rename from shardingsphere-test/shardingsphere-pipeline-test/src/test/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm
rename to shardingsphere-test/shardingsphere-pipeline-test/src/test/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm
index 72730b0..f75e6a7 100644
--- a/shardingsphere-test/shardingsphere-pipeline-test/src/test/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobCompletionDetectAlgorithm
+++ b/shardingsphere-test/shardingsphere-pipeline-test/src/test/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.detect.JobCompletionDetectAlgorithm
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.shardingsphere.data.pipeline.core.fixture.FixtureRuleAlteredJobCompletionDetectAlgorithm
+org.apache.shardingsphere.data.pipeline.core.fixture.FixtureJobCompletionDetectAlgorithm