You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2022/05/15 06:36:09 UTC
[shardingsphere] branch master updated: Related underlying modules change trigger scaling IT (#17664)
This is an automated email from the ASF dual-hosted git repository.
zhangliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new d08be86520e Related underlying modules change trigger scaling IT (#17664)
d08be86520e is described below
commit d08be86520ebc9792920f231cb850771ea3e352b
Author: Hongsheng Zhong <sa...@126.com>
AuthorDate: Sun May 15 14:36:03 2022 +0800
Related underlying modules change trigger scaling IT (#17664)
* Unify data pipeline SPI impl package
* Related underlying modules change trigger scaling IT
* Unit test
* Fix wrong column in IT
---
.github/workflows/it-scaling.yml | 28 +++++++++++-----------
.../pipeline}/ShardingRuleAlteredDetector.java | 2 +-
...hardingRuleAlteredJobConfigurationPreparer.java | 2 +-
...ta.pipeline.spi.rulealtered.RuleAlteredDetector | 2 +-
...rulealtered.RuleAlteredJobConfigurationPreparer | 2 +-
.../ShardingSpherePipelineDataSourceCreator.java | 2 +-
...re.datasource.creator.PipelineDataSourceCreator | 2 +-
.../pipeline/core/importer/AbstractImporter.java | 5 +++-
.../ingest/dumper/AbstractInventoryDumper.java | 2 +-
.../src/test/resources/env/common/command.xml | 4 ++--
.../core/importer/AbstractImporterTest.java | 4 ++--
.../sharding/ShardingRuleAlteredDetectorTest.java | 2 +-
12 files changed, 30 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/it-scaling.yml b/.github/workflows/it-scaling.yml
index 48087e9c4d7..95748c2c17a 100644
--- a/.github/workflows/it-scaling.yml
+++ b/.github/workflows/it-scaling.yml
@@ -22,14 +22,14 @@ on:
branches: [ master, dev ]
paths:
- '.github/workflows/it-scaling.yml'
- - 'shardingsphere-infra/shardingsphere-infra-common/src/main/**/datasource/**'
- - 'shardingsphere-kernel/shardingsphere-data-pipeline/**'
- - 'shardingsphere-kernel/shardingsphere-data-pipeline/**/src/test/**'
+ - 'shardingsphere-infra/shardingsphere-infra-common/src/main/**'
+ - 'shardingsphere-mode/**/src/main/**'
+ - 'shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/**'
+ - '**/*-distsql*/**/src/main/**'
+ - 'shardingsphere-kernel/shardingsphere-data-pipeline/**/src/main/**'
- 'shardingsphere-features/**/src/main/**/scaling/**'
- - 'shardingsphere-features/**/src/main/**/sharding/schedule/**'
- - 'shardingsphere-features/**/src/main/resources/META-INF/services/*.data.pipeline.*'
- - 'shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/**/*Pipeline*.java'
- - 'shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/resources/META-INF/services/*.data.pipeline.*'
+ - '**/src/main/**/data/pipeline/**'
+ - '**/src/main/resources/META-INF/services/*.data.pipeline.*'
- '!shardingsphere-test/**'
- 'shardingsphere-test/shardingsphere-integration-test/**'
- '!shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/**'
@@ -38,14 +38,14 @@ on:
branches: [ master ]
paths:
- '.github/workflows/it-scaling.yml'
- - 'shardingsphere-infra/shardingsphere-infra-common/src/main/**/datasource/**'
- - 'shardingsphere-kernel/shardingsphere-data-pipeline/**'
- - 'shardingsphere-kernel/shardingsphere-data-pipeline/**/src/test/**'
+ - 'shardingsphere-infra/shardingsphere-infra-common/src/main/**'
+ - 'shardingsphere-mode/**/src/main/**'
+ - 'shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/**'
+ - '**/*-distsql*/**/src/main/**'
+ - 'shardingsphere-kernel/shardingsphere-data-pipeline/**/src/main/**'
- 'shardingsphere-features/**/src/main/**/scaling/**'
- - 'shardingsphere-features/**/src/main/**/sharding/schedule/**'
- - 'shardingsphere-features/**/src/main/resources/META-INF/services/*.data.pipeline.*'
- - 'shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/**/*Pipeline*.java'
- - 'shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/resources/META-INF/services/*.data.pipeline.*'
+ - '**/src/main/**/data/pipeline/**'
+ - '**/src/main/resources/META-INF/services/*.data.pipeline.*'
- '!shardingsphere-test/**'
- 'shardingsphere-test/shardingsphere-integration-test/**'
- '!shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-suite/**'
diff --git a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/schedule/ShardingRuleAlteredDetector.java b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/data/pipeline/ShardingRuleAlteredDetector.java
similarity index 99%
rename from shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/schedule/ShardingRuleAlteredDetector.java
rename to shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/data/pipeline/ShardingRuleAlteredDetector.java
index 9e08f96fdb3..c6a73a5cd14 100644
--- a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/schedule/ShardingRuleAlteredDetector.java
+++ b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/data/pipeline/ShardingRuleAlteredDetector.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.sharding.schedule;
+package org.apache.shardingsphere.sharding.data.pipeline;
import org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredDetector;
import org.apache.shardingsphere.infra.config.RuleConfiguration;
diff --git a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/schedule/ShardingRuleAlteredJobConfigurationPreparer.java b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/data/pipeline/ShardingRuleAlteredJobConfigurationPreparer.java
similarity index 99%
rename from shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/schedule/ShardingRuleAlteredJobConfigurationPreparer.java
rename to shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/data/pipeline/ShardingRuleAlteredJobConfigurationPreparer.java
index bb5123abda4..3a96c8cd560 100644
--- a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/schedule/ShardingRuleAlteredJobConfigurationPreparer.java
+++ b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/java/org/apache/shardingsphere/sharding/data/pipeline/ShardingRuleAlteredJobConfigurationPreparer.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.sharding.schedule;
+package org.apache.shardingsphere.sharding.data.pipeline;
import com.google.common.base.Joiner;
import com.google.common.collect.Maps;
diff --git a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredDetector b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredDetector
index 707be3d24ae..2edfb14df15 100644
--- a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredDetector
+++ b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredDetector
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.shardingsphere.sharding.schedule.ShardingRuleAlteredDetector
+org.apache.shardingsphere.sharding.data.pipeline.ShardingRuleAlteredDetector
diff --git a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobConfigurationPreparer b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobConfigurationPreparer
index 6cdedbb762a..297de28d29f 100644
--- a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobConfigurationPreparer
+++ b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.spi.rulealtered.RuleAlteredJobConfigurationPreparer
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.shardingsphere.sharding.schedule.ShardingRuleAlteredJobConfigurationPreparer
+org.apache.shardingsphere.sharding.data.pipeline.ShardingRuleAlteredJobConfigurationPreparer
diff --git a/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/org/apache/shardingsphere/driver/config/datasource/ShardingSpherePipelineDataSourceCreator.java b/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/org/apache/shardingsphere/driver/data/pipeline/datasource/creator/ShardingSpherePipelineDataSourceCreator.java
similarity index 97%
rename from shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/org/apache/shardingsphere/driver/config/datasource/ShardingSpherePipelineDataSourceCreator.java
rename to shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/org/apache/shardingsphere/driver/data/pipeline/datasource/creator/ShardingSpherePipelineDataSourceCreator.java
index 2a55ac9936b..b6d0b1d003e 100644
--- a/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/org/apache/shardingsphere/driver/config/datasource/ShardingSpherePipelineDataSourceCreator.java
+++ b/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/java/org/apache/shardingsphere/driver/data/pipeline/datasource/creator/ShardingSpherePipelineDataSourceCreator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.driver.config.datasource;
+package org.apache.shardingsphere.driver.data.pipeline.datasource.creator;
import org.apache.shardingsphere.driver.api.ShardingSphereDataSourceFactory;
import org.apache.shardingsphere.infra.config.algorithm.ShardingSphereAlgorithmConfiguration;
diff --git a/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.core.datasource.creator.PipelineDataSourceCreator b/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.core.datasource.creator.PipelineDataSourceCreator
index 34321754006..6a071fa2ffb 100644
--- a/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.core.datasource.creator.PipelineDataSourceCreator
+++ b/shardingsphere-jdbc/shardingsphere-jdbc-core/src/main/resources/META-INF/services/org.apache.shardingsphere.data.pipeline.core.datasource.creator.PipelineDataSourceCreator
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.shardingsphere.driver.config.datasource.ShardingSpherePipelineDataSourceCreator
+org.apache.shardingsphere.driver.data.pipeline.datasource.creator.ShardingSpherePipelineDataSourceCreator
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporter.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporter.java
index 091660acbd8..7b8cd297293 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporter.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporter.java
@@ -191,7 +191,10 @@ public abstract class AbstractImporter extends AbstractLifecycleExecutor impleme
Column keyColumn = conditionColumns.get(i);
ps.setObject(updatedColumns.size() + i + 1, (keyColumn.isPrimaryKey() && keyColumn.isUpdated()) ? keyColumn.getOldValue() : keyColumn.getValue());
}
- ps.execute();
+ int updateCount = ps.executeUpdate();
+ if (1 != updateCount) {
+ log.warn("executeUpdate failed, updateCount={}, updateSql={}, updatedColumns={}, conditionColumns={}", updateCount, updateSql, updatedColumns, conditionColumns);
+ }
}
}
diff --git a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/AbstractInventoryDumper.java b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/AbstractInventoryDumper.java
index 8d05af12bbb..32afdf05012 100644
--- a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/AbstractInventoryDumper.java
+++ b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/dumper/AbstractInventoryDumper.java
@@ -181,7 +181,7 @@ public abstract class AbstractInventoryDumper extends AbstractLifecycleExecutor
break;
}
}
- if (PipelineJdbcUtils.isStringColumn(uniqueKeyDataType) && 0 == round % 50) {
+ if (0 == round % 50) {
log.info("dump, round={}, rowCount={}, maxUniqueKeyValue={}", round, rowCount, maxUniqueKeyValue);
}
return Optional.ofNullable(maxUniqueKeyValue);
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
index 5717c0086df..612057343bd 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/common/command.xml
@@ -41,7 +41,7 @@
DATANODES("ds_${0..1}.t_order_item_${0..1}"),
DATABASE_STRATEGY(TYPE=standard,SHARDING_COLUMN=user_id,SHARDING_ALGORITHM=database_inline),
TABLE_STRATEGY(TYPE=standard,SHARDING_COLUMN=order_id,SHARDING_ALGORITHM=t_order_item_inline),
- KEY_GENERATE_STRATEGY(COLUMN=order_item_id,TYPE(NAME=snowflake))
+ KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake))
)
</create-all-sharding-table-rule>
@@ -69,7 +69,7 @@
DATANODES("ds_${2..4}.t_order_item_${0..1}"),
DATABASE_STRATEGY(TYPE=standard,SHARDING_COLUMN=user_id,SHARDING_ALGORITHM=database_inline),
TABLE_STRATEGY(TYPE=standard,SHARDING_COLUMN=order_id,SHARDING_ALGORITHM=t_order_item_inline),
- KEY_GENERATE_STRATEGY(COLUMN=order_item_id,TYPE(NAME=snowflake)))
+ KEY_GENERATE_STRATEGY(COLUMN=order_id,TYPE(NAME=snowflake)))
</alter-sharding-table-rule>
<auto-alter-table-rule>
diff --git a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporterTest.java b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporterTest.java
index 5e913f9a0cd..6b1b19fabea 100644
--- a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporterTest.java
+++ b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/data/pipeline/core/importer/AbstractImporterTest.java
@@ -125,7 +125,7 @@ public final class AbstractImporterTest {
verify(preparedStatement).setObject(2, "UPDATE");
verify(preparedStatement).setObject(3, 1);
verify(preparedStatement).setObject(4, 10);
- verify(preparedStatement).execute();
+ verify(preparedStatement).executeUpdate();
}
@Test
@@ -140,7 +140,7 @@ public final class AbstractImporterTest {
inOrder.verify(preparedStatement).setObject(3, "UPDATE");
inOrder.verify(preparedStatement).setObject(4, 1);
inOrder.verify(preparedStatement).setObject(5, 10);
- inOrder.verify(preparedStatement).execute();
+ inOrder.verify(preparedStatement).executeUpdate();
}
private DataRecord getUpdatePrimaryKeyDataRecord() {
diff --git a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/sharding/ShardingRuleAlteredDetectorTest.java b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/sharding/ShardingRuleAlteredDetectorTest.java
index bfdd1e9a26f..d24a4e5176b 100644
--- a/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/sharding/ShardingRuleAlteredDetectorTest.java
+++ b/shardingsphere-test/shardingsphere-pipeline-test/src/test/java/org/apache/shardingsphere/sharding/ShardingRuleAlteredDetectorTest.java
@@ -21,7 +21,7 @@ import org.apache.shardingsphere.data.pipeline.api.datasource.config.impl.Shardi
import org.apache.shardingsphere.data.pipeline.core.util.ConfigurationFileUtil;
import org.apache.shardingsphere.infra.yaml.config.pojo.YamlRuleConfiguration;
import org.apache.shardingsphere.sharding.api.config.ShardingRuleConfiguration;
-import org.apache.shardingsphere.sharding.schedule.ShardingRuleAlteredDetector;
+import org.apache.shardingsphere.sharding.data.pipeline.ShardingRuleAlteredDetector;
import org.junit.Test;
import java.util.Collection;