You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2025/12/29 11:35:15 UTC

(shardingsphere) branch master updated: Improve InventoryPositionCalculator to support BigInteger (#37573)

This is an automated email from the ASF dual-hosted git repository.

zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git


The following commit(s) were added to refs/heads/master by this push:
     new efcc0097085 Improve InventoryPositionCalculator to support BigInteger (#37573)
efcc0097085 is described below

commit efcc00970850d469b4016badec17e027e888ed5c
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Mon Dec 29 19:35:09 2025 +0800

    Improve InventoryPositionCalculator to support BigInteger (#37573)
    
    * Improve InventoryPositionCalculator to support BigInteger
---
 .../calculator/InventoryDataSparsenessCalculator.java   |  9 +++++----
 .../position/InventoryPositionCalculator.java           |  3 ++-
 .../estimated/InventoryPositionEstimatedCalculator.java | 17 ++++++++++-------
 .../InventoryDataSparsenessCalculatorTest.java          | 14 ++++++++------
 .../InventoryPositionEstimatedCalculatorTest.java       | 17 +++++++++--------
 5 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculator.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculator.java
index 0026122f621..36837b7f468 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculator.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculator.java
@@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.shardingsphere.data.pipeline.core.ingest.dumper.inventory.query.Range;
 
 import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.math.RoundingMode;
 
 /**
@@ -43,12 +44,12 @@ public final class InventoryDataSparsenessCalculator {
      * @param uniqueKeyValuesRange unique key values range
      * @return true if sparse
      */
-    public static boolean isIntegerUniqueKeyDataSparse(final long tableRecordsCount, final Range<Long> uniqueKeyValuesRange) {
+    public static boolean isIntegerUniqueKeyDataSparse(final long tableRecordsCount, final Range<BigInteger> uniqueKeyValuesRange) {
         boolean result = false;
-        Long lowerBound = uniqueKeyValuesRange.getLowerBound();
-        Long upperBound = uniqueKeyValuesRange.getUpperBound();
+        BigInteger lowerBound = uniqueKeyValuesRange.getLowerBound();
+        BigInteger upperBound = uniqueKeyValuesRange.getUpperBound();
         if (tableRecordsCount >= EXACT_SPLITTING_RECORDS_COUNT_THRESHOLD && null != lowerBound && null != upperBound) {
-            BigDecimal multiple = BigDecimal.valueOf(upperBound).subtract(BigDecimal.valueOf(lowerBound)).add(BigDecimal.ONE)
+            BigDecimal multiple = new BigDecimal(upperBound).subtract(new BigDecimal(lowerBound)).add(BigDecimal.ONE)
                     .divide(BigDecimal.valueOf(tableRecordsCount), 2, RoundingMode.HALF_UP);
             if (multiple.compareTo(MULTIPLE_THRESHOLD) >= 0) {
                 log.info("Table is sparse for integer unique key, table records count: {}, unique key values range: {}, multiple: {}", tableRecordsCount, uniqueKeyValuesRange, multiple);
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/InventoryPositionCalculator.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/InventoryPositionCalculator.java
index 42a9b781fd8..7b7ec222453 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/InventoryPositionCalculator.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/InventoryPositionCalculator.java
@@ -32,6 +32,7 @@ import org.apache.shardingsphere.database.connector.core.metadata.database.metad
 import org.apache.shardingsphere.database.connector.core.type.DatabaseTypeRegistry;
 import org.apache.shardingsphere.infra.metadata.database.schema.QualifiedTable;
 
+import java.math.BigInteger;
 import java.util.Collections;
 import java.util.List;
 
@@ -70,7 +71,7 @@ public final class InventoryPositionCalculator {
     
     private List<IngestPosition> getIntegerPositions() {
         String uniqueKey = uniqueKeyColumns.get(0).getName();
-        Range<Long> uniqueKeyValuesRange = InventoryPositionEstimatedCalculator.getIntegerUniqueKeyValuesRange(qualifiedTable, uniqueKey, dataSource);
+        Range<BigInteger> uniqueKeyValuesRange = InventoryPositionEstimatedCalculator.getIntegerUniqueKeyValuesRange(qualifiedTable, uniqueKey, dataSource);
         if (InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(tableRecordsCount, uniqueKeyValuesRange)) {
             return InventoryPositionExactCalculator.getPositions(qualifiedTable, uniqueKey, shardingSize, dataSource, new IntegerPositionHandler());
         }
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculator.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculator.java
index 901b16cffb8..0fd2663af9f 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculator.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculator.java
@@ -28,6 +28,7 @@ import org.apache.shardingsphere.data.pipeline.core.ingest.position.type.pk.type
 import org.apache.shardingsphere.data.pipeline.core.sqlbuilder.sql.PipelinePrepareSQLBuilder;
 import org.apache.shardingsphere.infra.metadata.database.schema.QualifiedTable;
 
+import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -52,7 +53,7 @@ public final class InventoryPositionEstimatedCalculator {
      * @return unique key values range
      * @throws SplitPipelineJobByUniqueKeyException if an error occurs while getting unique key values range
      */
-    public static Range<Long> getIntegerUniqueKeyValuesRange(final QualifiedTable qualifiedTable, final String uniqueKey, final PipelineDataSource dataSource) {
+    public static Range<BigInteger> getIntegerUniqueKeyValuesRange(final QualifiedTable qualifiedTable, final String uniqueKey, final PipelineDataSource dataSource) {
         PipelinePrepareSQLBuilder pipelineSQLBuilder = new PipelinePrepareSQLBuilder(dataSource.getDatabaseType());
         String sql = pipelineSQLBuilder.buildUniqueKeyMinMaxValuesSQL(qualifiedTable.getSchemaName(), qualifiedTable.getTableName(), uniqueKey);
         try (
@@ -60,7 +61,9 @@ public final class InventoryPositionEstimatedCalculator {
                 Statement statement = connection.createStatement();
                 ResultSet resultSet = statement.executeQuery(sql)) {
             resultSet.next();
-            return Range.closed(resultSet.getLong(1), resultSet.getLong(2));
+            BigDecimal lowerBound = resultSet.getBigDecimal(1);
+            BigDecimal upperBound = resultSet.getBigDecimal(2);
+            return Range.closed(null == lowerBound ? null : lowerBound.toBigInteger(), null == upperBound ? null : upperBound.toBigInteger());
         } catch (final SQLException ex) {
             throw new SplitPipelineJobByUniqueKeyException(qualifiedTable.getTableName(), uniqueKey, ex);
         }
@@ -74,16 +77,16 @@ public final class InventoryPositionEstimatedCalculator {
      * @param shardingSize sharding size
      * @return positions
      */
-    public static List<IngestPosition> getIntegerPositions(final long tableRecordsCount, final Range<Long> uniqueKeyValuesRange, final long shardingSize) {
-        Long lowerBound = uniqueKeyValuesRange.getLowerBound();
-        Long upperBound = uniqueKeyValuesRange.getUpperBound();
+    public static List<IngestPosition> getIntegerPositions(final long tableRecordsCount, final Range<BigInteger> uniqueKeyValuesRange, final long shardingSize) {
+        BigInteger lowerBound = uniqueKeyValuesRange.getLowerBound();
+        BigInteger upperBound = uniqueKeyValuesRange.getUpperBound();
         if (0 == tableRecordsCount || null == lowerBound || null == upperBound) {
             return Collections.singletonList(new IntegerPrimaryKeyIngestPosition(null, null));
         }
         List<IngestPosition> result = new LinkedList<>();
         long splitCount = tableRecordsCount / shardingSize + (tableRecordsCount % shardingSize > 0 ? 1 : 0);
-        BigInteger stepSize = BigInteger.valueOf(upperBound).subtract(BigInteger.valueOf(lowerBound)).divide(BigInteger.valueOf(splitCount));
-        IntegerRangeSplittingIterator rangeIterator = new IntegerRangeSplittingIterator(BigInteger.valueOf(lowerBound), BigInteger.valueOf(upperBound), stepSize);
+        BigInteger stepSize = upperBound.subtract(lowerBound).divide(BigInteger.valueOf(splitCount));
+        IntegerRangeSplittingIterator rangeIterator = new IntegerRangeSplittingIterator(lowerBound, upperBound, stepSize);
         while (rangeIterator.hasNext()) {
             Range<BigInteger> range = rangeIterator.next();
             result.add(new IntegerPrimaryKeyIngestPosition(range.getLowerBound().longValue(), range.getUpperBound().longValue()));
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculatorTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculatorTest.java
index 6979188719d..b88e1fa59ec 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculatorTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/InventoryDataSparsenessCalculatorTest.java
@@ -20,6 +20,8 @@ package org.apache.shardingsphere.data.pipeline.core.preparer.inventory.calculat
 import org.apache.shardingsphere.data.pipeline.core.ingest.dumper.inventory.query.Range;
 import org.junit.jupiter.api.Test;
 
+import java.math.BigInteger;
+
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -27,11 +29,11 @@ class InventoryDataSparsenessCalculatorTest {
     
     @Test
     void assertIsIntegerUniqueKeyDataSparse() {
-        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1L, Range.closed(1L, 1000000L)));
-        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(null, 1000000L)));
-        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(1L, null)));
-        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(1L, 1494000L)));
-        assertTrue(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(1L, 1495000L)));
-        assertTrue(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(1L, 1500000L)));
+        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1L, Range.closed(BigInteger.ONE, BigInteger.valueOf(1000000L))));
+        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(null, BigInteger.valueOf(1000000L))));
+        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(BigInteger.ONE, null)));
+        assertFalse(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(BigInteger.ONE, BigInteger.valueOf(1494000L))));
+        assertTrue(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(BigInteger.ONE, BigInteger.valueOf(1495000L))));
+        assertTrue(InventoryDataSparsenessCalculator.isIntegerUniqueKeyDataSparse(1000000L, Range.closed(BigInteger.ONE, BigInteger.valueOf(1500000L))));
     }
 }
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculatorTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculatorTest.java
index 7fd1b129f4b..f16feb375af 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculatorTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/calculator/position/estimated/InventoryPositionEstimatedCalculatorTest.java
@@ -22,6 +22,7 @@ import org.apache.shardingsphere.data.pipeline.core.ingest.position.IngestPositi
 import org.apache.shardingsphere.data.pipeline.core.ingest.position.type.pk.type.IntegerPrimaryKeyIngestPosition;
 import org.junit.jupiter.api.Test;
 
+import java.math.BigInteger;
 import java.util.List;
 
 import static org.hamcrest.CoreMatchers.is;
@@ -32,7 +33,7 @@ class InventoryPositionEstimatedCalculatorTest {
     
     @Test
     void assertGetIntegerPositions() {
-        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(200L, Range.closed(1L, 600L), 100L);
+        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(200L, Range.closed(BigInteger.ONE, BigInteger.valueOf(600L)), 100L);
         assertThat(actualPositions.size(), is(2));
         assertPosition(actualPositions.get(0), new IntegerPrimaryKeyIngestPosition(1L, 300L));
         assertPosition(actualPositions.get(1), new IntegerPrimaryKeyIngestPosition(301L, 600L));
@@ -46,7 +47,7 @@ class InventoryPositionEstimatedCalculatorTest {
     
     @Test
     void assertGetIntegerPositionsWithZeroTotalRecordsCount() {
-        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(0L, Range.closed(0L, 1L), 1L);
+        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(0L, Range.closed(BigInteger.ZERO, BigInteger.ONE), 1L);
         assertThat(actualPositions.size(), is(1));
         assertPosition(actualPositions.get(0), new IntegerPrimaryKeyIngestPosition(null, null));
     }
@@ -60,7 +61,7 @@ class InventoryPositionEstimatedCalculatorTest {
     
     @Test
     void assertGetIntegerPositionsWithTheSameMinMax() {
-        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(200L, Range.closed(5L, 5L), 100L);
+        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(200L, Range.closed(BigInteger.valueOf(5L), BigInteger.valueOf(5L)), 100L);
         assertThat(actualPositions.size(), is(1));
         assertPosition(actualPositions.get(0), new IntegerPrimaryKeyIngestPosition(5L, 5L));
     }
@@ -69,11 +70,11 @@ class InventoryPositionEstimatedCalculatorTest {
     void assertGetIntegerPositionsOverflow() {
         long tableRecordsCount = Long.MAX_VALUE - 1L;
         long shardingSize = tableRecordsCount / 2L;
-        long minimum = Long.MIN_VALUE + 1L;
-        long maximum = Long.MAX_VALUE;
-        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(tableRecordsCount, Range.closed(minimum, maximum), shardingSize);
+        BigInteger lowerBound = BigInteger.valueOf(Long.MIN_VALUE + 1L);
+        BigInteger upperBound = BigInteger.valueOf(Long.MAX_VALUE);
+        List<IngestPosition> actualPositions = InventoryPositionEstimatedCalculator.getIntegerPositions(tableRecordsCount, Range.closed(lowerBound, upperBound), shardingSize);
         assertThat(actualPositions.size(), is(2));
-        assertPosition(actualPositions.get(0), new IntegerPrimaryKeyIngestPosition(minimum, 0L));
-        assertPosition(actualPositions.get(1), new IntegerPrimaryKeyIngestPosition(1L, maximum));
+        assertPosition(actualPositions.get(0), new IntegerPrimaryKeyIngestPosition(lowerBound.longValue(), 0L));
+        assertPosition(actualPositions.get(1), new IntegerPrimaryKeyIngestPosition(1L, upperBound.longValue()));
     }
 }