You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2022/05/18 03:05:47 UTC
[shardingsphere] branch master updated: Add fast failed of scaling it, print more logs (#17766)
This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 2383fa6e5bd Add fast failed of scaling it, print more logs (#17766)
2383fa6e5bd is described below
commit 2383fa6e5bdec87639cc7ef823a61da8332ad261
Author: azexcy <10...@users.noreply.github.com>
AuthorDate: Wed May 18 11:05:41 2022 +0800
Add fast failed of scaling it, print more logs (#17766)
* Add more log of scaling it.
* Method replace
---
.github/workflows/it-scaling.yml | 2 +-
.../data/pipeline/cases/base/BaseITCase.java | 2 +
.../cases/general/PostgreSQLGeneralScalingIT.java | 5 +-
.../data/pipeline/util/TableCrudUtil.java | 74 ----------------------
.../src/test/resources/logback-test.xml | 3 -
5 files changed, 6 insertions(+), 80 deletions(-)
diff --git a/.github/workflows/it-scaling.yml b/.github/workflows/it-scaling.yml
index 4763e8b7ce9..985a738227f 100644
--- a/.github/workflows/it-scaling.yml
+++ b/.github/workflows/it-scaling.yml
@@ -58,7 +58,7 @@ concurrency:
cancel-in-progress: true
env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.javadoc.skip=true -Drat.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Djacoco.skip=true
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.javadoc.skip=true -Drat.skip=true -Dcheckstyle.skip=true -Dspotless.apply.skip=true -Djacoco.skip=true -Dfailsafe.skipAfterFailureCount=1
jobs:
scaling-it-test:
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
index ea77fa946a4..6a6cd6b44ea 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
@@ -250,6 +250,7 @@ public abstract class BaseITCase {
assertThat(actualStatusMap.values().stream().filter(StringUtils::isNotBlank).collect(Collectors.toSet()).size(), is(1));
jdbcTemplate.execute(String.format("STOP SCALING SOURCE WRITING %s", jobId));
List<Map<String, Object>> checkScalingResults = jdbcTemplate.queryForList(String.format("CHECK SCALING %s BY TYPE (NAME=DATA_MATCH)", jobId));
+ log.info("checkScalingResults: {}", checkScalingResults);
for (Map<String, Object> entry : checkScalingResults) {
assertTrue(Boolean.parseBoolean(entry.get("records_content_matched").toString()));
}
@@ -258,6 +259,7 @@ public abstract class BaseITCase {
TimeUnit.SECONDS.sleep(2);
List<Map<String, Object>> previewResults = jdbcTemplate.queryForList("PREVIEW SELECT COUNT(1) FROM t_order");
Set<Object> originalSources = previewResults.stream().map(each -> each.get("data_source_name")).collect(Collectors.toSet());
+ log.info("originalSources: {}", originalSources);
assertThat(originalSources, is(new HashSet<>(Arrays.asList("ds_2", "ds_3", "ds_4"))));
}
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
index 254f0505bb9..dcb0b58747d 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
@@ -24,8 +24,8 @@ import org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseT
import org.apache.shardingsphere.integration.data.pipeline.cases.base.BaseExtraSQLITCase;
import org.apache.shardingsphere.integration.data.pipeline.cases.task.PostgreSQLIncrementTask;
import org.apache.shardingsphere.integration.data.pipeline.env.IntegrationTestEnvironment;
+import org.apache.shardingsphere.integration.data.pipeline.framework.helper.ScalingCaseHelper;
import org.apache.shardingsphere.integration.data.pipeline.framework.param.ScalingParameterized;
-import org.apache.shardingsphere.integration.data.pipeline.util.TableCrudUtil;
import org.apache.shardingsphere.sharding.algorithm.keygen.SnowflakeKeyGenerateAlgorithm;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -78,7 +78,8 @@ public final class PostgreSQLGeneralScalingIT extends BaseExtraSQLITCase {
bindingShardingRule();
createOrderTable();
createOrderItemTable();
- Pair<List<Object[]>, List<Object[]>> dataPair = TableCrudUtil.generatePostgresSQLInsertDataList(3000);
+ SnowflakeKeyGenerateAlgorithm keyGenerateAlgorithm = new SnowflakeKeyGenerateAlgorithm();
+ Pair<List<Object[]>, List<Object[]>> dataPair = ScalingCaseHelper.generateFullInsertData(keyGenerateAlgorithm, parameterized.getDatabaseType(), 3000);
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrder(), dataPair.getLeft());
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrderItem(), dataPair.getRight());
startIncrementTask(new PostgreSQLIncrementTask(getJdbcTemplate(), new SnowflakeKeyGenerateAlgorithm(), "test", true));
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/util/TableCrudUtil.java b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/util/TableCrudUtil.java
deleted file mode 100644
index 293e40d380a..00000000000
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/util/TableCrudUtil.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.shardingsphere.integration.data.pipeline.util;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.shardingsphere.sharding.algorithm.keygen.SnowflakeKeyGenerateAlgorithm;
-
-import java.math.BigDecimal;
-import java.nio.charset.StandardCharsets;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ThreadLocalRandom;
-
-public final class TableCrudUtil {
-
- private static final SnowflakeKeyGenerateAlgorithm SNOWFLAKE_GENERATE = new SnowflakeKeyGenerateAlgorithm();
-
- /**
- * Generate MySQL insert data, contains full fields.
- *
- * @param insertRows insert rows
- * @return insert data list
- */
- public static Pair<List<Object[]>, List<Object[]>> generateMySQLInsertDataList(final int insertRows) {
- if (insertRows < 0) {
- return Pair.of(null, null);
- }
- List<Object[]> orderData = new ArrayList<>(insertRows);
- List<Object[]> orderItemData = new ArrayList<>(insertRows);
- ThreadLocalRandom current = ThreadLocalRandom.current();
- for (int i = 0; i <= insertRows; i++) {
- int orderId = current.nextInt(0, 6);
- int userId = current.nextInt(0, 6);
- orderData.add(new Object[]{SNOWFLAKE_GENERATE.generateKey(), orderId, userId, "varchar" + i, (byte) 1, new Timestamp(System.currentTimeMillis()), new Timestamp(System.currentTimeMillis()),
- "hello".getBytes(StandardCharsets.UTF_8), null, new BigDecimal("100.00"), "test", Math.random(), "{}", current.nextInt(0, 10000000)});
- orderItemData.add(new Object[]{SNOWFLAKE_GENERATE.generateKey(), orderId, userId, "SUCCESS"});
- }
- return Pair.of(orderData, orderItemData);
- }
-
- /**
- * Generate PostgreSQL insert data, contains full fields.
- *
- * @param insertRows insert rows
- * @return insert data
- */
- public static Pair<List<Object[]>, List<Object[]>> generatePostgresSQLInsertDataList(final int insertRows) {
- List<Object[]> orderData = new ArrayList<>(insertRows);
- List<Object[]> orderItemData = new ArrayList<>(insertRows);
- for (int i = 1; i <= insertRows; i++) {
- int orderId = ThreadLocalRandom.current().nextInt(0, 6);
- int userId = ThreadLocalRandom.current().nextInt(0, 6);
- orderData.add(new Object[]{SNOWFLAKE_GENERATE.generateKey(), orderId, userId, "OK"});
- orderItemData.add(new Object[]{SNOWFLAKE_GENERATE.generateKey(), orderId, userId, "SUCCESS"});
- }
- return Pair.of(orderData, orderItemData);
- }
-}
diff --git a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/logback-test.xml b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/logback-test.xml
index 3ee302a72df..0b3462ef748 100644
--- a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/logback-test.xml
+++ b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/logback-test.xml
@@ -22,9 +22,6 @@
<pattern>[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %logger{36} - %msg%n</pattern>
</encoder>
</appender>
- <logger name="org.apache.shardingsphere" level="WARN" additivity="false">
- <appender-ref ref="console" />
- </logger>
<logger name="org.springframework.jdbc.core.JdbcTemplate" level="DEBUG" additivity="false">
<appender-ref ref="console" />
</logger>