You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by GitBox <gi...@apache.org> on 2022/05/13 12:55:58 UTC
[GitHub] [shardingsphere] sandynz commented on a diff in pull request #17637: Refactor Scaling IT code, prepare support multiple scenarios
sandynz commented on code in PR #17637:
URL: https://github.com/apache/shardingsphere/pull/17637#discussion_r872352172
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseTaskRunnable.java:
##########
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.integration.data.pipeline.cases.base;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
+import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+import java.time.Instant;
+import java.util.concurrent.ThreadLocalRandom;
+
+@Getter
+@AllArgsConstructor
+public abstract class BaseTaskRunnable implements Runnable {
+
+ protected static final ThreadLocalRandom RANDOM = ThreadLocalRandom.current();
Review Comment:
It's strange to set as `static` for `ThreadLocalRandom`, it's used in different threads and each thread get a new random but not the same random.
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseMySQLITCase.java:
##########
@@ -15,54 +15,61 @@
* limitations under the License.
*/
-package org.apache.shardingsphere.integration.data.pipeline.cases.mysql;
+package org.apache.shardingsphere.integration.data.pipeline.cases.base;
+import lombok.Getter;
import lombok.SneakyThrows;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import org.apache.shardingsphere.integration.data.pipeline.cases.BaseITCase;
import org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
-import org.apache.shardingsphere.integration.data.pipeline.cases.postgresql.BasePostgreSQLITCase;
+import org.apache.shardingsphere.integration.data.pipeline.cases.mysql.MySQLIncrementTaskRunnable;
+import org.apache.shardingsphere.integration.data.pipeline.framework.helper.SQLHelper;
import org.apache.shardingsphere.integration.data.pipeline.framework.param.ScalingParameterized;
-import org.apache.shardingsphere.integration.data.pipeline.util.TableCrudUtil;
+import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
import javax.xml.bind.JAXB;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-import java.util.List;
-import java.util.Objects;
import java.util.Properties;
public abstract class BaseMySQLITCase extends BaseITCase {
- protected static final DatabaseType DATABASE = new MySQLDatabaseType();
+ protected static final DatabaseType DATABASE_TYPE = new MySQLDatabaseType();
private final ExtraSQLCommand extraSQLCommand;
+ @Getter
+ private final SQLHelper sqlHelper;
+
public BaseMySQLITCase(final ScalingParameterized parameterized) {
super(parameterized);
- extraSQLCommand = JAXB.unmarshal(Objects.requireNonNull(BasePostgreSQLITCase.class.getClassLoader().getResource(parameterized.getParentPath() + "/sql.xml")), ExtraSQLCommand.class);
- initTableAndData();
+ extraSQLCommand = JAXB.unmarshal(BaseMySQLITCase.class.getClassLoader().getResource(parameterized.getScenario()), ExtraSQLCommand.class);
+ sqlHelper = new SQLHelper(DATABASE_TYPE, extraSQLCommand, getJdbcTemplate());
}
- @SneakyThrows({SQLException.class, InterruptedException.class})
- protected void initTableAndData() {
+ @SneakyThrows(SQLException.class)
+ protected void addResource() {
Properties queryProps = createQueryProperties();
// TODO if use jdbcurl like "jdbc:mysql:localhost:3307/sharding_db", will throw exception show "Datasource or ShardingSphere rule does not exist"
try (Connection connection = DriverManager.getConnection(JDBC_URL_APPENDER.appendQueryProperties(getComposedContainer().getProxyJdbcUrl(""), queryProps), "root", "root")) {
connection.createStatement().execute("USE sharding_db");
addResource(connection);
}
- initShardingRule();
- setIncreaseTaskThread(new Thread(new MySQLIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand)));
- getJdbcTemplate().execute(extraSQLCommand.getCreateTableOrder());
- getJdbcTemplate().execute(extraSQLCommand.getCreateTableOrderItem());
+ }
+
+ protected void startIncrementTask(final KeyGenerateAlgorithm keyGenerateAlgorithm) {
+ setIncreaseTaskThread(new Thread(new MySQLIncrementTaskRunnable(getJdbcTemplate(), extraSQLCommand, keyGenerateAlgorithm)));
getIncreaseTaskThread().start();
- Pair<List<Object[]>, List<Object[]>> dataPair = TableCrudUtil.generateMySQLInsertDataList(3000);
- getJdbcTemplate().batchUpdate(extraSQLCommand.getFullInsertOrder(), dataPair.getLeft());
- getJdbcTemplate().batchUpdate(extraSQLCommand.getInsertOrderItem(), dataPair.getRight());
+ }
+
+ /**
+ * Add no use table, to test part of the table.
+ */
+ protected void addNoUseTable() {
+ getJdbcTemplate().execute("CREATE TABLE no_use(id int(11) NOT NULL)");
+ getJdbcTemplate().execute("INSERT INTO no_use(id) values (1)");
}
Review Comment:
Seems there's no `no_use` table sharding rule, it won't be handled as partial tables scaling.
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/scenario/ScalingScenario.java:
##########
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.integration.data.pipeline.cases.scenario;
+
+import com.google.common.collect.Sets;
+
+import java.util.Collection;
+
+/**
+ * Manual scaling scenario.
+ */
+public final class ScalingScenario {
+
+ public static final String SCENARIO_SUFFIX = "sql.xml";
+
+ /**
+ * manual scenario list.
+ *
+ * @return scenario list
+ */
+ public static Collection<String> listScenario() {
+ return Sets.newHashSet("snowflake_primary_key");
Review Comment:
`integer_primary_key` might be better.
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/framework/helper/SQLHelper.java:
##########
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.integration.data.pipeline.framework.helper;
+
+import lombok.AllArgsConstructor;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.shardingsphere.infra.database.type.DatabaseType;
+import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
+import org.apache.shardingsphere.infra.database.type.dialect.OpenGaussDatabaseType;
+import org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseType;
+import org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
+import org.apache.shardingsphere.integration.data.pipeline.util.TableCrudUtil;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+import java.util.List;
+
+/**
+ * SQL helper.
+ */
+@AllArgsConstructor
+public final class SQLHelper {
Review Comment:
Could `SQLHelper` class name be more dedicated?
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/mysql/MySQLIncrementTaskRunnable.java:
##########
@@ -17,63 +17,49 @@
package org.apache.shardingsphere.integration.data.pipeline.cases.mysql;
-import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.shardingsphere.integration.data.pipeline.cases.base.BaseTaskRunnable;
import org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
-import org.apache.shardingsphere.integration.data.pipeline.util.TableCrudUtil;
+import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
import org.springframework.jdbc.core.JdbcTemplate;
-import java.sql.SQLException;
-import java.time.Instant;
import java.util.LinkedList;
import java.util.List;
@Slf4j
-@AllArgsConstructor
-public final class MySQLIncrementTaskRunnable implements Runnable {
+public final class MySQLIncrementTaskRunnable extends BaseTaskRunnable {
- private final JdbcTemplate jdbcTemplate;
-
- private final ExtraSQLCommand extraSQLCommand;
+ public MySQLIncrementTaskRunnable(final JdbcTemplate jdbcTemplate, final ExtraSQLCommand extraSQLCommand, final KeyGenerateAlgorithm keyGenerateAlgorithm) {
+ super(jdbcTemplate, extraSQLCommand, keyGenerateAlgorithm);
+ }
@Override
public void run() {
int executeCount = 0;
- List<Long> newPrimaryKeys = new LinkedList<>();
- try {
- while (executeCount < 20 && !Thread.currentThread().isInterrupted()) {
- newPrimaryKeys.add(insertOrderAndOrderItem());
- if (newPrimaryKeys.size() % 2 == 0) {
- deleteOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
- } else {
- updateOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
- }
- executeCount++;
- log.info("Increment task runnable execute successfully.");
+ List<Object> newOrderPrimaryKeys = new LinkedList<>();
+ List<Object> newOrderItemPrimaryKeys = new LinkedList<>();
+ while (executeCount < 20 && !Thread.currentThread().isInterrupted()) {
+ newOrderPrimaryKeys.add(insertOrder());
+ newOrderItemPrimaryKeys.add(insertOrderItem());
+ if (newOrderPrimaryKeys.size() % 2 == 0) {
+ deleteOrderByPrimaryKey(newOrderPrimaryKeys.get(newOrderPrimaryKeys.size() - 1));
+ deleteOrderItemByPrimaryKey(newOrderItemPrimaryKeys.get(newOrderItemPrimaryKeys.size() - 1));
+ } else {
+ updateOrderByPrimaryKey(newOrderPrimaryKeys.get(newOrderPrimaryKeys.size() - 1));
+ updateOrderItemByPrimaryKey(newOrderItemPrimaryKeys.get(newOrderItemPrimaryKeys.size() - 1));
}
Review Comment:
Looks it could be simplified:
1, Are `newOrderPrimaryKeys` and `newOrderItemPrimaryKeys` necessary?
2, Are `newOrderPrimaryKeys.get(newOrderPrimaryKeys.size() - 1)` and `newOrderItemPrimaryKeys.get(newOrderItemPrimaryKeys.size() - 1)` necessary?
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/scenario/ScalingScenario.java:
##########
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.integration.data.pipeline.cases.scenario;
+
+import com.google.common.collect.Sets;
+
+import java.util.Collection;
+
+/**
+ * Manual scaling scenario.
+ */
+public final class ScalingScenario {
+
+ public static final String SCENARIO_SUFFIX = "sql.xml";
+
+ /**
+ * manual scenario list.
Review Comment:
First character could be uppercase in javadoc.
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/mysql/MySQLManualScalingIT.java:
##########
@@ -57,20 +55,35 @@ public static Collection<ScalingParameterized> getParameters() {
if (Strings.isNullOrEmpty(version)) {
continue;
}
- result.add(new ScalingParameterized(DATABASE, version, "env/scenario/manual/mysql"));
+ for (String scenario : ScalingScenario.listScenario()) {
+ result.add(new ScalingParameterized(DATABASE_TYPE, version, Joiner.on("/").join("env/scenario/manual/mysql", scenario, ScalingScenario.SCENARIO_SUFFIX)));
+ }
}
return result;
}
+ @Before
+ public void setUp() {
+ addResource();
+ initShardingAlgorithm();
+ // TODO wait for algorithm init
+ ThreadUtil.sleep(2000);
Review Comment:
Does it need `20` seconds? Is there any way to wait on dynamic seconds within a maximum seconds?
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/scenario/manual/postgresql/uuid_primary_key/sql.xml:
##########
@@ -0,0 +1,52 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<command>
+ <create-table-order>
+ CREATE TABLE test.t_order (
+ id varchar(255) NOT NULL,
+ order_id int4 NOT NULL,
+ user_id int NOT NULL,
+ PRIMARY KEY (id)
+ )
+ </create-table-order>
+
+ <create-table-order-item>
+ CREATE TABLE test.t_order_item (
+ item_id int8 NOT NULL,
+ order_id int4 NOT NULL,
+ user_id int8 NOT NULL,
+ status varchar(50),
Review Comment:
Could we share the same `sql.xml` between databases for `uuid_primary_key`?
And it's better to use `text_primary_key` etc.
##########
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/mysql/MySQLIncrementTaskRunnable.java:
##########
@@ -17,63 +17,49 @@
package org.apache.shardingsphere.integration.data.pipeline.cases.mysql;
-import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.shardingsphere.integration.data.pipeline.cases.base.BaseTaskRunnable;
import org.apache.shardingsphere.integration.data.pipeline.cases.command.ExtraSQLCommand;
-import org.apache.shardingsphere.integration.data.pipeline.util.TableCrudUtil;
+import org.apache.shardingsphere.sharding.spi.KeyGenerateAlgorithm;
import org.springframework.jdbc.core.JdbcTemplate;
-import java.sql.SQLException;
-import java.time.Instant;
import java.util.LinkedList;
import java.util.List;
@Slf4j
-@AllArgsConstructor
-public final class MySQLIncrementTaskRunnable implements Runnable {
+public final class MySQLIncrementTaskRunnable extends BaseTaskRunnable {
- private final JdbcTemplate jdbcTemplate;
-
- private final ExtraSQLCommand extraSQLCommand;
+ public MySQLIncrementTaskRunnable(final JdbcTemplate jdbcTemplate, final ExtraSQLCommand extraSQLCommand, final KeyGenerateAlgorithm keyGenerateAlgorithm) {
+ super(jdbcTemplate, extraSQLCommand, keyGenerateAlgorithm);
+ }
@Override
public void run() {
int executeCount = 0;
- List<Long> newPrimaryKeys = new LinkedList<>();
- try {
- while (executeCount < 20 && !Thread.currentThread().isInterrupted()) {
- newPrimaryKeys.add(insertOrderAndOrderItem());
- if (newPrimaryKeys.size() % 2 == 0) {
- deleteOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
- } else {
- updateOrderAndOrderItem(newPrimaryKeys.get(newPrimaryKeys.size() - 1));
- }
- executeCount++;
- log.info("Increment task runnable execute successfully.");
+ List<Object> newOrderPrimaryKeys = new LinkedList<>();
+ List<Object> newOrderItemPrimaryKeys = new LinkedList<>();
+ while (executeCount < 20 && !Thread.currentThread().isInterrupted()) {
+ newOrderPrimaryKeys.add(insertOrder());
+ newOrderItemPrimaryKeys.add(insertOrderItem());
+ if (newOrderPrimaryKeys.size() % 2 == 0) {
+ deleteOrderByPrimaryKey(newOrderPrimaryKeys.get(newOrderPrimaryKeys.size() - 1));
+ deleteOrderItemByPrimaryKey(newOrderItemPrimaryKeys.get(newOrderItemPrimaryKeys.size() - 1));
+ } else {
+ updateOrderByPrimaryKey(newOrderPrimaryKeys.get(newOrderPrimaryKeys.size() - 1));
+ updateOrderItemByPrimaryKey(newOrderItemPrimaryKeys.get(newOrderItemPrimaryKeys.size() - 1));
}
- } catch (final SQLException ex) {
- log.error("IncrementTaskThread error", ex);
- throw new RuntimeException(ex);
Review Comment:
How exception will be handled if there's exception thrown.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@shardingsphere.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org