You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2023/10/16 12:44:44 UTC
[shardingsphere] branch master updated: Refactor sequentialFlush at PipelineDataSourceSink of Pipeline (#28734)
This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 559f12154a4 Refactor sequentialFlush at PipelineDataSourceSink of Pipeline (#28734)
559f12154a4 is described below
commit 559f12154a40f3bf6944c84e808f488e9f36242d
Author: Xinze Guo <10...@users.noreply.github.com>
AuthorDate: Mon Oct 16 20:44:33 2023 +0800
Refactor sequentialFlush at PipelineDataSourceSink of Pipeline (#28734)
* Refactor sequentialFlush at PipelineDataSourceSink
* Refactor doFlush
* Improve and add TODO
---
.../core/importer/sink/PipelineDataSourceSink.java | 52 +++++-----------------
1 file changed, 11 insertions(+), 41 deletions(-)
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java
index ca3a5165184..0995292e0db 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java
@@ -135,7 +135,10 @@ public final class PipelineDataSourceSink implements PipelineSink {
private void doFlush(final DataSource dataSource, final List<DataRecord> buffer) throws SQLException {
try (Connection connection = dataSource.getConnection()) {
- connection.setAutoCommit(false);
+ boolean enableTransaction = buffer.size() > 1;
+ if (enableTransaction) {
+ connection.setAutoCommit(false);
+ }
switch (buffer.get(0).getType()) {
case IngestDataChangeType.INSERT:
if (null != rateLimitAlgorithm) {
@@ -158,45 +161,12 @@ public final class PipelineDataSourceSink implements PipelineSink {
default:
break;
}
- connection.commit();
- }
- }
-
- private void doFlush(final Connection connection, final List<DataRecord> buffer) {
- // TODO it's better use transaction, but execute delete maybe not effect when open transaction of PostgreSQL sometimes
- for (DataRecord each : buffer) {
- try {
- doFlush(connection, each);
- } catch (final SQLException ex) {
- throw new PipelineImporterJobWriteException(String.format("Write failed, record=%s", each), ex);
+ if (enableTransaction) {
+ connection.commit();
}
}
}
- private void doFlush(final Connection connection, final DataRecord dataRecord) throws SQLException {
- switch (dataRecord.getType()) {
- case IngestDataChangeType.INSERT:
- if (null != rateLimitAlgorithm) {
- rateLimitAlgorithm.intercept(JobOperationType.INSERT, 1);
- }
- executeBatchInsert(connection, Collections.singletonList(dataRecord));
- break;
- case IngestDataChangeType.UPDATE:
- if (null != rateLimitAlgorithm) {
- rateLimitAlgorithm.intercept(JobOperationType.UPDATE, 1);
- }
- executeUpdate(connection, dataRecord);
- break;
- case IngestDataChangeType.DELETE:
- if (null != rateLimitAlgorithm) {
- rateLimitAlgorithm.intercept(JobOperationType.DELETE, 1);
- }
- executeBatchDelete(connection, Collections.singletonList(dataRecord));
- break;
- default:
- }
- }
-
private void executeBatchInsert(final Connection connection, final List<DataRecord> dataRecords) throws SQLException {
DataRecord dataRecord = dataRecords.get(0);
String insertSql = importSQLBuilder.buildInsertSQL(getSchemaName(dataRecord.getTableName()), dataRecord);
@@ -279,11 +249,11 @@ public final class PipelineDataSourceSink implements PipelineSink {
}
private void sequentialFlush(final DataSource dataSource, final List<DataRecord> buffer) {
- if (buffer.isEmpty()) {
- return;
- }
- try (Connection connection = dataSource.getConnection()) {
- doFlush(connection, buffer);
+ // TODO it's better use transaction, but execute delete maybe not effect when open transaction of PostgreSQL sometimes
+ try {
+ for (DataRecord each : buffer) {
+ doFlush(dataSource, Collections.singletonList(each));
+ }
} catch (final SQLException ex) {
throw new PipelineImporterJobWriteException(ex);
}