You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2023/02/11 03:48:09 UTC
[shardingsphere] branch master updated: Review and improve pipeline code (#24113)
This is an automated email from the ASF dual-hosted git repository.
zhaojinchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 709e1c3a687 Review and improve pipeline code (#24113)
709e1c3a687 is described below
commit 709e1c3a687c8629fd71eec77b4e438ab61681cc
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Sat Feb 11 11:48:01 2023 +0800
Review and improve pipeline code (#24113)
* Improve pipeline E2E
* Fix PREPARE_SUCCESS not set in jobItemContext
* Remove 18032 error code in doc
* Remove unused config file scaling_job_config.yaml
* Recover createSourceSchema
---
.../user-manual/error-code/sql-error-code.cn.md | 1 -
.../user-manual/error-code/sql-error-code.en.md | 1 -
.../migration/prepare/MigrationJobPreparer.java | 1 +
.../pipeline/cases/base/PipelineBaseE2EIT.java | 4 ++--
.../scaling/rule_alter/scaling_job_config.yaml | 26 ----------------------
5 files changed, 3 insertions(+), 30 deletions(-)
diff --git a/docs/document/content/user-manual/error-code/sql-error-code.cn.md b/docs/document/content/user-manual/error-code/sql-error-code.cn.md
index 345a4145b02..dfc190e1c07 100644
--- a/docs/document/content/user-manual/error-code/sql-error-code.cn.md
+++ b/docs/document/content/user-manual/error-code/sql-error-code.cn.md
@@ -109,7 +109,6 @@ SQL 错误码以标准的 SQL State，Vendor Code 和详细错误信息提供，
| HY000 | 18020 | Failed to get DDL for table \`%s\`. |
| 42S01 | 18030 | Duplicate storage unit names \`%s\`. |
| 42S02 | 18031 | Storage units names \`%s\` do not exist. |
-| 0A000 | 18032 | Unsupported data type \`%s\` of unique key for pipeline job. |
| HY000 | 18050 | Before data record is \`%s\`, after data record is \`%s\`. |
| 08000 | 18051 | Data check table \`%s\` failed. |
| 0A000 | 18052 | Unsupported pipeline database type \`%s\`. |
diff --git a/docs/document/content/user-manual/error-code/sql-error-code.en.md b/docs/document/content/user-manual/error-code/sql-error-code.en.md
index 7809c334f74..75eef7e1e2b 100644
--- a/docs/document/content/user-manual/error-code/sql-error-code.en.md
+++ b/docs/document/content/user-manual/error-code/sql-error-code.en.md
@@ -109,7 +109,6 @@ SQL error codes provide by standard `SQL State`, `Vendor Code` and `Reason`, whi
| HY000 | 18020 | Failed to get DDL for table \`%s\`. |
| 42S01 | 18030 | Duplicate storage unit names \`%s\`. |
| 42S02 | 18031 | Storage units names \`%s\` do not exist. |
-| 0A000 | 18032 | Unsupported data type \`%s\` of unique key for pipeline job. |
| HY000 | 18050 | Before data record is \`%s\`, after data record is \`%s\`. |
| 08000 | 18051 | Data check table \`%s\` failed. |
| 0A000 | 18052 | Unsupported pipeline database type \`%s\`. |
diff --git a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
index 700eae2112d..ba4d7c9821c 100644
--- a/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
+++ b/kernel/data-pipeline/scenario/migration/src/main/java/org/apache/shardingsphere/data/pipeline/scenario/migration/prepare/MigrationJobPreparer.java
@@ -110,6 +110,7 @@ public final class MigrationJobPreparer {
prepareAndCheckTarget(jobItemContext);
// TODO Loop insert zookeeper performance is not good
for (int i = 0; i <= jobItemContext.getJobConfig().getJobShardingCount(); i++) {
+ jobItemContext.setStatus(JobStatus.PREPARE_SUCCESS);
jobAPI.updateJobItemStatus(jobConfig.getJobId(), i, JobStatus.PREPARE_SUCCESS);
}
}
diff --git a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java
index 9aad57418bb..8cdfb189541 100644
--- a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java
+++ b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/base/PipelineBaseE2EIT.java
@@ -148,10 +148,10 @@ public abstract class PipelineBaseE2EIT {
return;
}
try {
- connection.createStatement().execute(String.format("DROP DATABASE %s", PROXY_DATABASE));
+ connection.createStatement().execute(String.format("DROP DATABASE IF EXISTS %s", PROXY_DATABASE));
ThreadUtil.sleep(2, TimeUnit.SECONDS);
} catch (final SQLException ex) {
- log.warn("Drop proxy database failed, maybe it's not exist. error msg={}", ex.getMessage());
+ log.warn("Drop proxy database failed, error={}", ex.getMessage());
}
}
diff --git a/test/it/pipeline/src/test/resources/scaling/rule_alter/scaling_job_config.yaml b/test/it/pipeline/src/test/resources/scaling/rule_alter/scaling_job_config.yaml
deleted file mode 100644
index b22d2140252..00000000000
--- a/test/it/pipeline/src/test/resources/scaling/rule_alter/scaling_job_config.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-jobName: 0130317c30317c3054317c6c6f6769635f6462
-jobParameter: |
- handleConfig:
- jobId: 0130317c30317c3054317c6c6f6769635f6462
- jobShardingDataNodes:
- - t_order:ds_0.t_order_0,ds_0.t_order_1
- workflowConfig:
- databaseName: logic_db
-shardingTotalCount: 1