You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by az...@apache.org on 2023/11/17 10:50:10 UTC
(shardingsphere) branch master updated: Review and improve pipeline code (#29068)
This is an automated email from the ASF dual-hosted git repository.
azexin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 3bb34d7820f Review and improve pipeline code (#29068)
3bb34d7820f is described below
commit 3bb34d7820f8eee047775deb1785d4149eaad51e
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Fri Nov 17 18:50:03 2023 +0800
Review and improve pipeline code (#29068)
* Improve isIncrementalSupported check on migration preparer
* Improve integer matching on consistency check
* Add CaseInsensitiveQualifiedTableTest
---
.../DataConsistencyCheckUtils.java | 24 +++++++++++++++++++++-
...RecordSingleTableInventoryCalculatedResult.java | 5 ++---
.../CaseInsensitiveQualifiedTableTest.java} | 23 +++++++++++++--------
.../DataConsistencyCheckUtilsTest.java | 11 ++++++++++
.../migration/prepare/MigrationJobPreparer.java | 7 +++++--
5 files changed, 55 insertions(+), 15 deletions(-)
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java
index adb79c32554..0aa0844db66 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java
@@ -65,9 +65,20 @@ public final class DataConsistencyCheckUtils {
return true;
}
+ /**
+ * Whether column values are matched or not.
+ *
+ * @param equalsBuilder equals builder
+ * @param thisColumnValue this column value
+ * @param thatColumnValue that column value
+ * @return true if matched, otherwise false
+ */
@SneakyThrows(SQLException.class)
- private static boolean isMatched(final EqualsBuilder equalsBuilder, final Object thisColumnValue, final Object thatColumnValue) {
+ public static boolean isMatched(final EqualsBuilder equalsBuilder, final Object thisColumnValue, final Object thatColumnValue) {
equalsBuilder.reset();
+ if (isInteger(thisColumnValue) && isInteger(thatColumnValue)) {
+ return isIntegerEquals((Number) thisColumnValue, (Number) thatColumnValue);
+ }
if (thisColumnValue instanceof SQLXML && thatColumnValue instanceof SQLXML) {
return ((SQLXML) thisColumnValue).getString().equals(((SQLXML) thatColumnValue).getString());
}
@@ -80,6 +91,17 @@ public final class DataConsistencyCheckUtils {
return equalsBuilder.append(thisColumnValue, thatColumnValue).isEquals();
}
+ private static boolean isInteger(final Object value) {
+ if (!(value instanceof Number)) {
+ return false;
+ }
+ return value instanceof Long || value instanceof Integer || value instanceof Short || value instanceof Byte;
+ }
+
+ private static boolean isIntegerEquals(final Number one, final Number another) {
+ return one.longValue() == another.longValue();
+ }
+
/**
* Check two BigDecimal whether equals or not.
*
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java
index 1db10f4352b..8bd104982f0 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java
@@ -26,7 +26,6 @@ import org.apache.shardingsphere.data.pipeline.core.consistencycheck.DataConsist
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Optional;
/**
@@ -66,12 +65,12 @@ public final class RecordSingleTableInventoryCalculatedResult implements SingleT
return false;
}
final RecordSingleTableInventoryCalculatedResult that = (RecordSingleTableInventoryCalculatedResult) o;
- if (recordsCount != that.recordsCount || !Objects.equals(maxUniqueKeyValue, that.maxUniqueKeyValue)) {
+ EqualsBuilder equalsBuilder = new EqualsBuilder();
+ if (recordsCount != that.recordsCount || !DataConsistencyCheckUtils.isMatched(equalsBuilder, maxUniqueKeyValue, that.maxUniqueKeyValue)) {
log.warn("Record count or max unique key value not match, recordCount1={}, recordCount2={}, maxUniqueKeyValue1={}, maxUniqueKeyValue2={}.",
recordsCount, that.recordsCount, maxUniqueKeyValue, that.maxUniqueKeyValue);
return false;
}
- EqualsBuilder equalsBuilder = new EqualsBuilder();
Iterator<Map<String, Object>> thisRecordsIterator = records.iterator();
Iterator<Map<String, Object>> thatRecordsIterator = that.records.iterator();
while (thisRecordsIterator.hasNext() && thatRecordsIterator.hasNext()) {
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/common/metadata/CaseInsensitiveQualifiedTableTest.java
similarity index 53%
copy from kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
copy to kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/common/metadata/CaseInsensitiveQualifiedTableTest.java
index 13a4fbba35e..c554dbfc998 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/common/metadata/CaseInsensitiveQualifiedTableTest.java
@@ -15,20 +15,25 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.data.pipeline.core.consistencycheck;
+package org.apache.shardingsphere.data.pipeline.common.metadata;
import org.junit.jupiter.api.Test;
-import java.math.BigDecimal;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-class DataConsistencyCheckUtilsTest {
+class CaseInsensitiveQualifiedTableTest {
+
+ @Test
+ void assertToString() {
+ CaseInsensitiveQualifiedTable actual = new CaseInsensitiveQualifiedTable(new CaseInsensitiveIdentifier(null), new CaseInsensitiveIdentifier("t_order"));
+ assertThat(actual.toString(), is("t_order"));
+ }
@Test
- void assertIsBigDecimalEquals() {
- BigDecimal one = BigDecimal.valueOf(3322, 1);
- BigDecimal another = BigDecimal.valueOf(33220, 2);
- assertTrue(DataConsistencyCheckUtils.isBigDecimalEquals(one, another));
+ void assertToStringThrowsNPE() {
+ CaseInsensitiveQualifiedTable actual = new CaseInsensitiveQualifiedTable(null, new CaseInsensitiveIdentifier("t_order"));
+ assertThrows(NullPointerException.class, actual::toString);
}
}
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
index 13a4fbba35e..01676117059 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
@@ -17,6 +17,7 @@
package org.apache.shardingsphere.data.pipeline.core.consistencycheck;
+import org.apache.commons.lang3.builder.EqualsBuilder;
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
@@ -25,6 +26,16 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class DataConsistencyCheckUtilsTest {
+ @Test
+ void assertIsIntegerEquals() {
+ EqualsBuilder equalsBuilder = new EqualsBuilder();
+ String value = "123";
+ Long longValue = Long.parseLong(value);
+ assertTrue(DataConsistencyCheckUtils.isMatched(equalsBuilder, longValue, Integer.parseInt(value)));
+ assertTrue(DataConsistencyCheckUtils.isMatched(equalsBuilder, longValue, Short.parseShort(value)));
+ assertTrue(DataConsistencyCheckUtils.isMatched(equalsBuilder, longValue, Byte.parseByte(value)));
+ }
+
@Test
void assertIsBigDecimalEquals() {
BigDecimal one = BigDecimal.valueOf(3322, 1);
diff --git a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
index e6ef884220a..cf73d15646b 100644
--- a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
+++ b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
@@ -101,9 +101,12 @@ public final class MigrationJobPreparer {
PipelineJobCenter.stop(jobItemContext.getJobId());
return;
}
- prepareIncremental(jobItemContext);
+ boolean isIncrementalSupported = PipelineJobPreparerUtils.isIncrementalSupported(jobItemContext.getJobConfig().getSourceDatabaseType());
+ if (isIncrementalSupported) {
+ prepareIncremental(jobItemContext);
+ }
initInventoryTasks(jobItemContext);
- if (PipelineJobPreparerUtils.isIncrementalSupported(jobItemContext.getJobConfig().getSourceDatabaseType())) {
+ if (isIncrementalSupported) {
initIncrementalTasks(jobItemContext);
if (jobItemContext.isStopping()) {
PipelineJobCenter.stop(jobItemContext.getJobId());