You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2022/06/11 05:30:48 UTC
[shardingsphere] branch master updated: Improve scaling it (#18288)
This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new bc232a908db Improve scaling it (#18288)
bc232a908db is described below
commit bc232a908db07cedcaa700a45761b876894f436f
Author: azexcy <10...@users.noreply.github.com>
AuthorDate: Sat Jun 11 13:30:34 2022 +0800
Improve scaling it (#18288)
- add create index of PostgreSQL
- add comment on of PostgreSQL
---
.../data/pipeline/cases/base/BaseExtraSQLITCase.java | 8 ++++++++
.../data/pipeline/cases/base/BaseITCase.java | 18 +++++++++++-------
.../data/pipeline/cases/command/ExtraSQLCommand.java | 7 +++++++
.../cases/general/PostgreSQLGeneralScalingIT.java | 1 +
.../test/resources/env/scenario/general/postgresql.xml | 8 ++++++++
5 files changed, 35 insertions(+), 7 deletions(-)
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java
index 30c084bf533..56fc0a8f4fb 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseExtraSQLITCase.java
@@ -22,6 +22,7 @@ import org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQ
import org.apache.shardingsphere.integration.data.pipeline.framework.param.ScalingParameterized;
import javax.xml.bind.JAXB;
+import java.util.List;
public abstract class BaseExtraSQLITCase extends BaseITCase {
@@ -42,6 +43,13 @@ public abstract class BaseExtraSQLITCase extends BaseITCase {
executeWithLog(extraSQLCommand.getCreateTableOrder());
}
+ protected void createTableIndexList() {
+ List<String> createTableIndexList = extraSQLCommand.getCreateTableIndexList();
+ for (String each : createTableIndexList) {
+ executeWithLog(each);
+ }
+ }
+
protected void createOrderItemTable() {
executeWithLog(extraSQLCommand.getCreateTableOrderItem());
}
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
index ef283f8e30b..70c2e03e20d 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
@@ -51,7 +51,6 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -248,13 +247,18 @@ public abstract class BaseITCase {
getIncreaseTaskThread().start();
}
+ protected void stopScalingSourceWriting(final String jobId) {
+ executeWithLog(String.format("STOP SCALING SOURCE WRITING %s", jobId));
+ }
+
+ protected void applyScaling(final String jobId) {
+ executeWithLog(String.format("APPLY SCALING %s", jobId));
+ }
+
protected void assertBeforeApplyScalingMetadataCorrectly() {
List<Map<String, Object>> previewResults = queryForListWithLog("PREVIEW SELECT COUNT(1) FROM t_order");
- Set<Object> actualSources = previewResults.stream().map(each -> each.get("actual_sql")).collect(Collectors.toSet());
assertThat("data_source_name name not correct, it's effective early, search watcher failed get more info",
previewResults.stream().map(each -> each.get("data_source_name")).collect(Collectors.toSet()), is(new HashSet<>(Arrays.asList("ds_0", "ds_1"))));
- assertThat("actual_sql not correct, it's effective early, search watcher failed get more info", actualSources,
- is(new HashSet<>(Collections.singletonList("SELECT COUNT(1) FROM t_order_0 UNION ALL SELECT COUNT(1) FROM t_order_1"))));
}
/**
@@ -270,7 +274,7 @@ public abstract class BaseITCase {
TimeUnit.SECONDS.sleep(4);
List<Map<String, Object>> scalingListMap = queryForListWithLog("SHOW SCALING LIST");
assertThat(scalingListMap.size(), is(1));
- Object jobId = scalingListMap.get(0).get("id");
+ String jobId = scalingListMap.get(0).get("id").toString();
log.info("jobId: {}", jobId);
Map<String, String> actualStatusMap = new HashMap<>(2, 1);
String showScalingStatus = String.format("SHOW SCALING STATUS %s", jobId);
@@ -297,7 +301,7 @@ public abstract class BaseITCase {
TimeUnit.SECONDS.sleep(2);
}
assertThat(actualStatusMap.values().stream().filter(StringUtils::isNotBlank).collect(Collectors.toSet()).size(), is(1));
- executeWithLog(String.format("STOP SCALING SOURCE WRITING %s", jobId));
+ stopScalingSourceWriting(jobId);
assertBeforeApplyScalingMetadataCorrectly();
List<Map<String, Object>> checkScalingResults = queryForListWithLog(String.format("CHECK SCALING %s BY TYPE (NAME=DATA_MATCH)", jobId));
log.info("checkScalingResults: {}", checkScalingResults);
@@ -305,7 +309,7 @@ public abstract class BaseITCase {
assertTrue(Boolean.parseBoolean(entry.get("records_content_matched").toString()));
}
assertBeforeApplyScalingMetadataCorrectly();
- executeWithLog(String.format("APPLY SCALING %s", jobId));
+ applyScaling(jobId);
// TODO make sure the scaling job was applied
TimeUnit.SECONDS.sleep(2);
List<Map<String, Object>> previewResults = queryForListWithLog("PREVIEW SELECT COUNT(1) FROM t_order");
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java
index 1b3788ca445..e2cc13be89e 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/command/ExtraSQLCommand.java
@@ -23,6 +23,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
@Data
@XmlRootElement(name = "command")
@@ -40,4 +41,10 @@ public final class ExtraSQLCommand {
@XmlElement(name = "full-insert-order-item")
private String fullInsertOrderItem;
+
+ @XmlElement(name = "create-table-index")
+ private List<String> createTableIndexList;
+
+ @XmlElement(name = "create-column-comment")
+ private List<String> createColumnCommentList;
}
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
index 3c96896d05a..c958ca897e0 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
@@ -76,6 +76,7 @@ public final class PostgreSQLGeneralScalingIT extends BaseExtraSQLITCase {
createSchema("test");
getCreateOrderWithItemSharingTableRule();
createOrderTable();
+ createTableIndexList();
createOrderItemTable();
SnowflakeKeyGenerateAlgorithm keyGenerateAlgorithm = new SnowflakeKeyGenerateAlgorithm();
Pair<List<Object[]>, List<Object[]>> dataPair = ScalingCaseHelper.generateFullInsertData(keyGenerateAlgorithm, parameterized.getDatabaseType(), 3000);
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/general/postgresql.xml b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/general/postgresql.xml
index 696eccedf5c..7659fd78bee 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/general/postgresql.xml
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/general/postgresql.xml
@@ -40,6 +40,14 @@
)
</create-table-order>
+ <create-table-index>
+ CREATE INDEX "idx_user_id" ON test.t_order ( user_id );
+ </create-table-index>
+
+ <create-column-comment>
+ COMMENT ON COLUMN test.t_order.user_id IS 'user id';
+ </create-column-comment>
+
<create-table-order-item>
CREATE TABLE test.t_order_item (
item_id int8 NOT NULL,