You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2022/08/02 04:29:28 UTC
[shardingsphere] branch master updated: Add more scaling IT cases. (#19744)
This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 929b4304558 Add more scaling IT cases. (#19744)
929b4304558 is described below
commit 929b43045581b2ae7a80e0b445647f1f32588198
Author: Xinze Guo <10...@users.noreply.github.com>
AuthorDate: Tue Aug 2 12:29:23 2022 +0800
Add more scaling IT cases. (#19744)
* Increase MySQL json length at scaling IT.
* Add total count check
* Fix ci
* Change check order
---
.../integration/data/pipeline/cases/base/BaseITCase.java | 9 +++++++++
.../data/pipeline/cases/general/MySQLGeneralScalingIT.java | 6 +++---
.../data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java | 3 ++-
.../data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java | 4 ++--
.../data/pipeline/framework/helper/ScalingCaseHelper.java | 6 +++++-
5 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
index e53c0880a90..10628d82044 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
@@ -23,6 +23,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
import org.apache.shardingsphere.data.pipeline.api.job.JobStatus;
import org.apache.shardingsphere.data.pipeline.core.util.ThreadUtil;
import org.apache.shardingsphere.infra.database.metadata.url.JdbcUrlAppender;
@@ -87,6 +88,8 @@ public abstract class BaseITCase {
protected static final Executor SCALING_EXECUTOR = Executors.newFixedThreadPool(5);
+ protected static final int TABLE_INIT_ROW_COUNT = 3000;
+
@Rule
@Getter(AccessLevel.NONE)
public ScalingWatcher scalingWatcher;
@@ -376,6 +379,12 @@ public abstract class BaseITCase {
assertThat(actualStatus, is(Collections.singleton(JobStatus.EXECUTE_INCREMENTAL_TASK.name())));
}
+ protected void assertGreaterThanInitTableInitRows(final int tableInitRows, final String schema) {
+ String countSQL = StringUtils.isBlank(schema) ? "SELECT COUNT(*) as count FROM t_order" : String.format("SELECT COUNT(*) as count FROM %s.t_order", schema);
+ Map<String, Object> actual = jdbcTemplate.queryForMap(countSQL);
+ assertTrue("actual count " + actual.get("count"), Integer.parseInt(actual.get("count").toString()) > TABLE_INIT_ROW_COUNT);
+ }
+
protected List<Map<String, Object>> showScalingStatus(final String jobId) {
return queryForListWithLog(String.format("SHOW SCALING STATUS %s", jobId));
}
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
index 1c6a3067caf..7b8cc7cf333 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
@@ -78,7 +78,7 @@ public final class MySQLGeneralScalingIT extends BaseExtraSQLITCase {
createOrderTable();
createOrderItemTable();
SnowflakeKeyGenerateAlgorithm keyGenerateAlgorithm = new SnowflakeKeyGenerateAlgorithm();
- for (int i = 0; i < 3; i++) {
+ for (int i = 0; i < TABLE_INIT_ROW_COUNT / 1000; i++) {
Pair<List<Object[]>, List<Object[]>> dataPair = ScalingCaseHelper.generateFullInsertData(keyGenerateAlgorithm, parameterized.getDatabaseType(), 1000);
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrder(), dataPair.getLeft());
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrderItem(), dataPair.getRight());
@@ -89,10 +89,10 @@ public final class MySQLGeneralScalingIT extends BaseExtraSQLITCase {
String jobId = getScalingJobId();
waitScalingFinished(jobId);
stopScaling(jobId);
- // TODO need netty leak fixed
- getJdbcTemplate().update("INSERT INTO t_order (id,order_id,user_id,status) VALUES (?, ?, ?, ?)", keyGenerateAlgorithm.generateKey(), 1, 1, "afterStopScaling");
+ getJdbcTemplate().update("INSERT INTO t_order (id,order_id,user_id,status,t_json) VALUES (?, ?, ?, ?, ?)", keyGenerateAlgorithm.generateKey(), 1, 1, "afterStopScaling", "{}");
startScaling(jobId);
assertCheckScalingSuccess(jobId);
+ assertGreaterThanInitTableInitRows(TABLE_INIT_ROW_COUNT, "");
applyScaling(jobId);
assertPreviewTableSuccess("t_order", Arrays.asList("ds_2", "ds_3", "ds_4"));
assertPreviewTableSuccess("t_order_item", Arrays.asList("ds_2", "ds_3", "ds_4"));
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
index b3391a33e4b..01c042b1078 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
@@ -84,7 +84,7 @@ public final class PostgreSQLGeneralScalingIT extends BaseExtraSQLITCase {
// createTableIndexList("test");
executeWithLog("COMMENT ON COLUMN test.t_order.user_id IS 'user id';");
SnowflakeKeyGenerateAlgorithm keyGenerateAlgorithm = new SnowflakeKeyGenerateAlgorithm();
- Pair<List<Object[]>, List<Object[]>> dataPair = ScalingCaseHelper.generateFullInsertData(keyGenerateAlgorithm, parameterized.getDatabaseType(), 3000);
+ Pair<List<Object[]>, List<Object[]>> dataPair = ScalingCaseHelper.generateFullInsertData(keyGenerateAlgorithm, parameterized.getDatabaseType(), TABLE_INIT_ROW_COUNT);
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrder(), dataPair.getLeft());
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrderItem(), dataPair.getRight());
addTargetResource();
@@ -96,6 +96,7 @@ public final class PostgreSQLGeneralScalingIT extends BaseExtraSQLITCase {
executeWithLog(String.format("INSERT INTO test.t_order (id,order_id,user_id,status) VALUES (%s, %s, %s, '%s')", keyGenerateAlgorithm.generateKey(), 1, 1, "afterStopScaling"));
startScaling(jobId);
assertCheckScalingSuccess(jobId);
+ assertGreaterThanInitTableInitRows(TABLE_INIT_ROW_COUNT, "test");
applyScaling(jobId);
assertPreviewTableSuccess("t_order", Arrays.asList("ds_2", "ds_3", "ds_4"));
assertPreviewTableSuccess("t_order_item", Arrays.asList("ds_2", "ds_3", "ds_4"));
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
index f8a12acafab..aec5cb698d6 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
@@ -87,8 +87,8 @@ public class TextPrimaryKeyScalingIT extends BaseExtraSQLITCase {
private void batchInsertOrder() {
UUIDKeyGenerateAlgorithm keyGenerateAlgorithm = new UUIDKeyGenerateAlgorithm();
- List<Object[]> orderData = new ArrayList<>(3000);
- for (int i = 1; i <= 3000; i++) {
+ List<Object[]> orderData = new ArrayList<>(TABLE_INIT_ROW_COUNT);
+ for (int i = 0; i < TABLE_INIT_ROW_COUNT; i++) {
orderData.add(new Object[]{keyGenerateAlgorithm.generateKey(), ThreadLocalRandom.current().nextInt(0, 6), ThreadLocalRandom.current().nextInt(0, 6), "OK"});
}
getJdbcTemplate().batchUpdate("INSERT INTO t_order (id,order_id,user_id,status) VALUES (?,?,?,?)", orderData);
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/helper/ScalingCaseHelper.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/helper/ScalingCaseHelper.java
index 03f02b63c88..a94af88e6e9 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/helper/ScalingCaseHelper.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/helper/ScalingCaseHelper.java
@@ -59,7 +59,7 @@ public final class ScalingCaseHelper {
Object[] addObjs = {keyGenerateAlgorithm.generateKey(), orderId, userId, generateString(6), randomInt, randomInt, randomInt,
randomUnsignedInt, randomUnsignedInt, randomUnsignedInt, randomUnsignedInt, generateFloat(), generateDouble(-1000, 100000),
BigDecimal.valueOf(generateDouble(1, 100)), now, now, now.toLocalDate(), now.toLocalTime(), null, "1", "t", "e", "s", "t", generateString(2), generateString(1),
- generateString(1), "1", "2", "{}"};
+ generateString(1), "1", "2", generateJsonString(4096)};
orderData.add(addObjs);
} else {
orderData.add(new Object[]{keyGenerateAlgorithm.generateKey(), orderId, userId, generateString(6), randomInt,
@@ -79,6 +79,10 @@ public final class ScalingCaseHelper {
return RandomStringUtils.randomAlphabetic(strLength);
}
+ private static String generateJsonString(final int strLength) {
+ return String.format("{\"test\":\"%s\"}", generateString(strLength));
+ }
+
private static float generateFloat() {
return ThreadLocalRandom.current().nextFloat();
}