You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by az...@apache.org on 2023/11/17 10:50:10 UTC

(shardingsphere) branch master updated: Review and improve pipeline code (#29068)

This is an automated email from the ASF dual-hosted git repository.

azexin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git


The following commit(s) were added to refs/heads/master by this push:
     new 3bb34d7820f Review and improve pipeline code (#29068)
3bb34d7820f is described below

commit 3bb34d7820f8eee047775deb1785d4149eaad51e
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Fri Nov 17 18:50:03 2023 +0800

    Review and improve pipeline code (#29068)
    
    * Improve isIncrementalSupported check on migration preparer
    
    * Improve integer matching on consistency check
    
    * Add CaseInsensitiveQualifiedTableTest
---
 .../DataConsistencyCheckUtils.java                 | 24 +++++++++++++++++++++-
 ...RecordSingleTableInventoryCalculatedResult.java |  5 ++---
 .../CaseInsensitiveQualifiedTableTest.java}        | 23 +++++++++++++--------
 .../DataConsistencyCheckUtilsTest.java             | 11 ++++++++++
 .../migration/prepare/MigrationJobPreparer.java    |  7 +++++--
 5 files changed, 55 insertions(+), 15 deletions(-)

diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java
index adb79c32554..0aa0844db66 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtils.java
@@ -65,9 +65,20 @@ public final class DataConsistencyCheckUtils {
         return true;
     }
     
+    /**
+     * Whether column values are matched or not.
+     *
+     * @param equalsBuilder equals builder
+     * @param thisColumnValue this column value
+     * @param thatColumnValue that column value
+     * @return true if matched, otherwise false
+     */
     @SneakyThrows(SQLException.class)
-    private static boolean isMatched(final EqualsBuilder equalsBuilder, final Object thisColumnValue, final Object thatColumnValue) {
+    public static boolean isMatched(final EqualsBuilder equalsBuilder, final Object thisColumnValue, final Object thatColumnValue) {
         equalsBuilder.reset();
+        if (isInteger(thisColumnValue) && isInteger(thatColumnValue)) {
+            return isIntegerEquals((Number) thisColumnValue, (Number) thatColumnValue);
+        }
         if (thisColumnValue instanceof SQLXML && thatColumnValue instanceof SQLXML) {
             return ((SQLXML) thisColumnValue).getString().equals(((SQLXML) thatColumnValue).getString());
         }
@@ -80,6 +91,17 @@ public final class DataConsistencyCheckUtils {
         return equalsBuilder.append(thisColumnValue, thatColumnValue).isEquals();
     }
     
+    private static boolean isInteger(final Object value) {
+        if (!(value instanceof Number)) {
+            return false;
+        }
+        return value instanceof Long || value instanceof Integer || value instanceof Short || value instanceof Byte;
+    }
+    
+    private static boolean isIntegerEquals(final Number one, final Number another) {
+        return one.longValue() == another.longValue();
+    }
+    
     /**
      * Check two BigDecimal whether equals or not.
      *
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java
index 1db10f4352b..8bd104982f0 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/result/RecordSingleTableInventoryCalculatedResult.java
@@ -26,7 +26,6 @@ import org.apache.shardingsphere.data.pipeline.core.consistencycheck.DataConsist
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 
 /**
@@ -66,12 +65,12 @@ public final class RecordSingleTableInventoryCalculatedResult implements SingleT
             return false;
         }
         final RecordSingleTableInventoryCalculatedResult that = (RecordSingleTableInventoryCalculatedResult) o;
-        if (recordsCount != that.recordsCount || !Objects.equals(maxUniqueKeyValue, that.maxUniqueKeyValue)) {
+        EqualsBuilder equalsBuilder = new EqualsBuilder();
+        if (recordsCount != that.recordsCount || !DataConsistencyCheckUtils.isMatched(equalsBuilder, maxUniqueKeyValue, that.maxUniqueKeyValue)) {
             log.warn("Record count or max unique key value not match, recordCount1={}, recordCount2={}, maxUniqueKeyValue1={}, maxUniqueKeyValue2={}.",
                     recordsCount, that.recordsCount, maxUniqueKeyValue, that.maxUniqueKeyValue);
             return false;
         }
-        EqualsBuilder equalsBuilder = new EqualsBuilder();
         Iterator<Map<String, Object>> thisRecordsIterator = records.iterator();
         Iterator<Map<String, Object>> thatRecordsIterator = that.records.iterator();
         while (thisRecordsIterator.hasNext() && thatRecordsIterator.hasNext()) {
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/common/metadata/CaseInsensitiveQualifiedTableTest.java
similarity index 53%
copy from kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
copy to kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/common/metadata/CaseInsensitiveQualifiedTableTest.java
index 13a4fbba35e..c554dbfc998 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/common/metadata/CaseInsensitiveQualifiedTableTest.java
@@ -15,20 +15,25 @@
  * limitations under the License.
  */
 
-package org.apache.shardingsphere.data.pipeline.core.consistencycheck;
+package org.apache.shardingsphere.data.pipeline.common.metadata;
 
 import org.junit.jupiter.api.Test;
 
-import java.math.BigDecimal;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-class DataConsistencyCheckUtilsTest {
+class CaseInsensitiveQualifiedTableTest {
+    
+    @Test
+    void assertToString() {
+        CaseInsensitiveQualifiedTable actual = new CaseInsensitiveQualifiedTable(new CaseInsensitiveIdentifier(null), new CaseInsensitiveIdentifier("t_order"));
+        assertThat(actual.toString(), is("t_order"));
+    }
     
     @Test
-    void assertIsBigDecimalEquals() {
-        BigDecimal one = BigDecimal.valueOf(3322, 1);
-        BigDecimal another = BigDecimal.valueOf(33220, 2);
-        assertTrue(DataConsistencyCheckUtils.isBigDecimalEquals(one, another));
+    void assertToStringThrowsNPE() {
+        CaseInsensitiveQualifiedTable actual = new CaseInsensitiveQualifiedTable(null, new CaseInsensitiveIdentifier("t_order"));
+        assertThrows(NullPointerException.class, actual::toString);
     }
 }
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
index 13a4fbba35e..01676117059 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/consistencycheck/DataConsistencyCheckUtilsTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.shardingsphere.data.pipeline.core.consistencycheck;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.junit.jupiter.api.Test;
 
 import java.math.BigDecimal;
@@ -25,6 +26,16 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 class DataConsistencyCheckUtilsTest {
     
+    @Test
+    void assertIsIntegerEquals() {
+        EqualsBuilder equalsBuilder = new EqualsBuilder();
+        String value = "123";
+        Long longValue = Long.parseLong(value);
+        assertTrue(DataConsistencyCheckUtils.isMatched(equalsBuilder, longValue, Integer.parseInt(value)));
+        assertTrue(DataConsistencyCheckUtils.isMatched(equalsBuilder, longValue, Short.parseShort(value)));
+        assertTrue(DataConsistencyCheckUtils.isMatched(equalsBuilder, longValue, Byte.parseByte(value)));
+    }
+    
     @Test
     void assertIsBigDecimalEquals() {
         BigDecimal one = BigDecimal.valueOf(3322, 1);
diff --git a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
index e6ef884220a..cf73d15646b 100644
--- a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
+++ b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
@@ -101,9 +101,12 @@ public final class MigrationJobPreparer {
             PipelineJobCenter.stop(jobItemContext.getJobId());
             return;
         }
-        prepareIncremental(jobItemContext);
+        boolean isIncrementalSupported = PipelineJobPreparerUtils.isIncrementalSupported(jobItemContext.getJobConfig().getSourceDatabaseType());
+        if (isIncrementalSupported) {
+            prepareIncremental(jobItemContext);
+        }
         initInventoryTasks(jobItemContext);
-        if (PipelineJobPreparerUtils.isIncrementalSupported(jobItemContext.getJobConfig().getSourceDatabaseType())) {
+        if (isIncrementalSupported) {
             initIncrementalTasks(jobItemContext);
             if (jobItemContext.isStopping()) {
                 PipelineJobCenter.stop(jobItemContext.getJobId());