You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2023/10/16 12:44:44 UTC

[shardingsphere] branch master updated: Refactor sequentialFlush at PipelineDataSourceSink of Pipeline (#28734)

This is an automated email from the ASF dual-hosted git repository.

zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git


The following commit(s) were added to refs/heads/master by this push:
     new 559f12154a4 Refactor sequentialFlush at PipelineDataSourceSink of Pipeline (#28734)
559f12154a4 is described below

commit 559f12154a40f3bf6944c84e808f488e9f36242d
Author: Xinze Guo <10...@users.noreply.github.com>
AuthorDate: Mon Oct 16 20:44:33 2023 +0800

    Refactor sequentialFlush at PipelineDataSourceSink of Pipeline (#28734)
    
    * Refactor sequentialFlush at PipelineDataSourceSink
    
    * Refactor doFlush
    
    * Improve and add TODO
---
 .../core/importer/sink/PipelineDataSourceSink.java | 52 +++++-----------------
 1 file changed, 11 insertions(+), 41 deletions(-)

diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java
index ca3a5165184..0995292e0db 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/sink/PipelineDataSourceSink.java
@@ -135,7 +135,10 @@ public final class PipelineDataSourceSink implements PipelineSink {
     
     private void doFlush(final DataSource dataSource, final List<DataRecord> buffer) throws SQLException {
         try (Connection connection = dataSource.getConnection()) {
-            connection.setAutoCommit(false);
+            boolean enableTransaction = buffer.size() > 1;
+            if (enableTransaction) {
+                connection.setAutoCommit(false);
+            }
             switch (buffer.get(0).getType()) {
                 case IngestDataChangeType.INSERT:
                     if (null != rateLimitAlgorithm) {
@@ -158,45 +161,12 @@ public final class PipelineDataSourceSink implements PipelineSink {
                 default:
                     break;
             }
-            connection.commit();
-        }
-    }
-    
-    private void doFlush(final Connection connection, final List<DataRecord> buffer) {
-        // TODO it's better use transaction, but execute delete maybe not effect when open transaction of PostgreSQL sometimes
-        for (DataRecord each : buffer) {
-            try {
-                doFlush(connection, each);
-            } catch (final SQLException ex) {
-                throw new PipelineImporterJobWriteException(String.format("Write failed, record=%s", each), ex);
+            if (enableTransaction) {
+                connection.commit();
             }
         }
     }
     
-    private void doFlush(final Connection connection, final DataRecord dataRecord) throws SQLException {
-        switch (dataRecord.getType()) {
-            case IngestDataChangeType.INSERT:
-                if (null != rateLimitAlgorithm) {
-                    rateLimitAlgorithm.intercept(JobOperationType.INSERT, 1);
-                }
-                executeBatchInsert(connection, Collections.singletonList(dataRecord));
-                break;
-            case IngestDataChangeType.UPDATE:
-                if (null != rateLimitAlgorithm) {
-                    rateLimitAlgorithm.intercept(JobOperationType.UPDATE, 1);
-                }
-                executeUpdate(connection, dataRecord);
-                break;
-            case IngestDataChangeType.DELETE:
-                if (null != rateLimitAlgorithm) {
-                    rateLimitAlgorithm.intercept(JobOperationType.DELETE, 1);
-                }
-                executeBatchDelete(connection, Collections.singletonList(dataRecord));
-                break;
-            default:
-        }
-    }
-    
     private void executeBatchInsert(final Connection connection, final List<DataRecord> dataRecords) throws SQLException {
         DataRecord dataRecord = dataRecords.get(0);
         String insertSql = importSQLBuilder.buildInsertSQL(getSchemaName(dataRecord.getTableName()), dataRecord);
@@ -279,11 +249,11 @@ public final class PipelineDataSourceSink implements PipelineSink {
     }
     
     private void sequentialFlush(final DataSource dataSource, final List<DataRecord> buffer) {
-        if (buffer.isEmpty()) {
-            return;
-        }
-        try (Connection connection = dataSource.getConnection()) {
-            doFlush(connection, buffer);
+        // TODO it's better use transaction, but execute delete maybe not effect when open transaction of PostgreSQL sometimes
+        try {
+            for (DataRecord each : buffer) {
+                doFlush(dataSource, Collections.singletonList(each));
+            }
         } catch (final SQLException ex) {
             throw new PipelineImporterJobWriteException(ex);
         }