You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by az...@apache.org on 2023/04/25 06:17:54 UTC

[shardingsphere] branch master updated: Reduce CDC fetching records waiting time (#25313)

This is an automated email from the ASF dual-hosted git repository.

azexin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git


The following commit(s) were added to refs/heads/master by this push:
     new 6ce0fe171af Reduce CDC fetching records waiting time (#25313)
6ce0fe171af is described below

commit 6ce0fe171af3f5829bada693e4eb76f07764eaa4
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Tue Apr 25 14:17:47 2023 +0800

    Reduce CDC fetching records waiting time (#25313)
    
    * Refactor PipelineChannel.fetchRecords
    
    * Reduce CDC fetching records waiting time
---
 .../data/pipeline/api/ingest/channel/PipelineChannel.java     |  6 ++++--
 .../data/pipeline/cdc/core/importer/SocketSinkImporter.java   |  3 ++-
 .../data/pipeline/core/importer/DataSourceImporter.java       |  3 ++-
 .../ingest/channel/memory/MultiplexMemoryPipelineChannel.java |  5 +++--
 .../ingest/channel/memory/SimpleMemoryPipelineChannel.java    |  5 +++--
 .../channel/memory/MultiplexMemoryPipelineChannelTest.java    |  2 +-
 .../pipeline/mysql/ingest/MySQLIncrementalDumperTest.java     | 11 ++++++-----
 .../pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java   |  3 ++-
 .../data/pipeline/core/importer/DataSourceImporterTest.java   |  8 ++++----
 9 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java b/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java
index deec0dee990..5fa2b8ce353 100644
--- a/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java
+++ b/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java
@@ -20,6 +20,7 @@ package org.apache.shardingsphere.data.pipeline.api.ingest.channel;
 import org.apache.shardingsphere.data.pipeline.api.ingest.record.Record;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Pipeline channel.
@@ -38,10 +39,11 @@ public interface PipelineChannel {
      * It might be blocked at most timeout seconds if available records count doesn't reach batch size.
      *
      * @param batchSize record batch size
-     * @param timeoutSeconds timeout(seconds)
+     * @param timeout timeout
+     * @param timeUnit time unit
      * @return record
      */
-    List<Record> fetchRecords(int batchSize, int timeoutSeconds);
+    List<Record> fetchRecords(int batchSize, int timeout, TimeUnit timeUnit);
     
     /**
      * Ack the last batch.
diff --git a/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java b/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java
index 2fe971b7818..e6dbc6e4d36 100644
--- a/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java
+++ b/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java
@@ -40,6 +40,7 @@ import org.apache.shardingsphere.data.pipeline.spi.ratelimit.JobRateLimitAlgorit
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 /**
@@ -79,7 +80,7 @@ public final class SocketSinkImporter extends AbstractLifecycleExecutor implemen
             importerConnector.sendIncrementalStartEvent(this, batchSize);
         }
         while (isRunning()) {
-            List<Record> records = channel.fetchRecords(batchSize, 3);
+            List<Record> records = channel.fetchRecords(batchSize, 500, TimeUnit.MILLISECONDS);
             if (null != records && !records.isEmpty()) {
                 List<Record> recordList = records.stream().filter(each -> !(each instanceof PlaceholderRecord)).collect(Collectors.toList());
                 try {
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java
index 4cc79efb8dd..83692765b88 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java
@@ -50,6 +50,7 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 /**
@@ -93,7 +94,7 @@ public final class DataSourceImporter extends AbstractLifecycleExecutor implemen
     protected void runBlocking() {
         int batchSize = importerConfig.getBatchSize() * 2;
         while (isRunning()) {
-            List<Record> records = channel.fetchRecords(batchSize, 3);
+            List<Record> records = channel.fetchRecords(batchSize, 3, TimeUnit.SECONDS);
             if (null != records && !records.isEmpty()) {
                 PipelineJobProgressUpdatedParameter updatedParam = flush(dataSourceManager.getDataSource(importerConfig.getDataSourceConfig()), records);
                 channel.ack(records);
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java
index a9e3cd49514..aa459ecd92f 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java
@@ -27,6 +27,7 @@ import org.apache.shardingsphere.data.pipeline.api.ingest.record.Record;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -67,8 +68,8 @@ public final class MultiplexMemoryPipelineChannel implements PipelineChannel {
     }
     
     @Override
-    public List<Record> fetchRecords(final int batchSize, final int timeoutSeconds) {
-        return findChannel().fetchRecords(batchSize, timeoutSeconds);
+    public List<Record> fetchRecords(final int batchSize, final int timeout, final TimeUnit timeUnit) {
+        return findChannel().fetchRecords(batchSize, timeout, timeUnit);
     }
     
     @Override
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java
index 6484aeb0b1f..e292f57d0e7 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Simple memory pipeline channel.
@@ -53,11 +54,11 @@ public final class SimpleMemoryPipelineChannel implements PipelineChannel {
     @SneakyThrows(InterruptedException.class)
     // TODO thread-safe?
     @Override
-    public List<Record> fetchRecords(final int batchSize, final int timeoutSeconds) {
+    public List<Record> fetchRecords(final int batchSize, final int timeout, final TimeUnit timeUnit) {
         List<Record> result = new ArrayList<>(batchSize);
         long start = System.currentTimeMillis();
         while (batchSize > queue.size()) {
-            if (timeoutSeconds * 1000L <= System.currentTimeMillis() - start) {
+            if (timeUnit.toMillis(timeout) <= System.currentTimeMillis() - start) {
                 break;
             }
             Thread.sleep(100L);
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java
index 1e99ca65fea..2ca5d558b16 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java
@@ -87,7 +87,7 @@ class MultiplexMemoryPipelineChannelTest {
     private static void fetch(final MultiplexMemoryPipelineChannel memoryChannel, final CountDownLatch countDownLatch) {
         int maxLoopCount = 10;
         for (int j = 1; j <= maxLoopCount; j++) {
-            List<Record> records = memoryChannel.fetchRecords(100, 1);
+            List<Record> records = memoryChannel.fetchRecords(100, 1, TimeUnit.SECONDS);
             memoryChannel.ack(records);
             records.forEach(each -> countDownLatch.countDown());
             if (!records.isEmpty() && records.get(records.size() - 1) instanceof FinishedRecord) {
diff --git a/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java b/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java
index c5f7ac75567..f6653c39c09 100644
--- a/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java
+++ b/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java
@@ -63,6 +63,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -150,7 +151,7 @@ class MySQLIncrementalDumperTest {
         rowsEvent.setAfterRows(Collections.singletonList(new Serializable[]{101, 1, "OK"}));
         Method method = MySQLIncrementalDumper.class.getDeclaredMethod("handleWriteRowsEvent", WriteRowsEvent.class, PipelineTableMetaData.class);
         Plugins.getMemberAccessor().invoke(method, incrementalDumper, rowsEvent, pipelineTableMetaData);
-        List<Record> actual = channel.fetchRecords(1, 0);
+        List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
         assertThat(actual.size(), is(1));
         assertThat(actual.get(0), instanceOf(DataRecord.class));
         assertThat(((DataRecord) actual.get(0)).getType(), is(IngestDataChangeType.INSERT));
@@ -180,7 +181,7 @@ class MySQLIncrementalDumperTest {
         rowsEvent.setAfterRows(Collections.singletonList(new Serializable[]{101, 1, "OK2"}));
         Method method = MySQLIncrementalDumper.class.getDeclaredMethod("handleUpdateRowsEvent", UpdateRowsEvent.class, PipelineTableMetaData.class);
         Plugins.getMemberAccessor().invoke(method, incrementalDumper, rowsEvent, pipelineTableMetaData);
-        List<Record> actual = channel.fetchRecords(1, 0);
+        List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
         assertThat(actual.size(), is(1));
         assertThat(actual.get(0), instanceOf(DataRecord.class));
         assertThat(((DataRecord) actual.get(0)).getType(), is(IngestDataChangeType.UPDATE));
@@ -205,7 +206,7 @@ class MySQLIncrementalDumperTest {
         rowsEvent.setBeforeRows(Collections.singletonList(new Serializable[]{101, 1, "OK"}));
         Method method = MySQLIncrementalDumper.class.getDeclaredMethod("handleDeleteRowsEvent", DeleteRowsEvent.class, PipelineTableMetaData.class);
         Plugins.getMemberAccessor().invoke(method, incrementalDumper, rowsEvent, pipelineTableMetaData);
-        List<Record> actual = channel.fetchRecords(1, 0);
+        List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
         assertThat(actual.size(), is(1));
         assertThat(actual.get(0), instanceOf(DataRecord.class));
         assertThat(((DataRecord) actual.get(0)).getType(), is(IngestDataChangeType.DELETE));
@@ -215,7 +216,7 @@ class MySQLIncrementalDumperTest {
     @Test
     void assertPlaceholderEvent() throws ReflectiveOperationException {
         Plugins.getMemberAccessor().invoke(MySQLIncrementalDumper.class.getDeclaredMethod("handleEvent", AbstractBinlogEvent.class), incrementalDumper, new PlaceholderEvent());
-        List<Record> actual = channel.fetchRecords(1, 0);
+        List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
         assertThat(actual.size(), is(1));
         assertThat(actual.get(0), instanceOf(PlaceholderRecord.class));
     }
@@ -225,7 +226,7 @@ class MySQLIncrementalDumperTest {
         WriteRowsEvent rowsEvent = new WriteRowsEvent();
         rowsEvent.setDatabaseName("unknown_database");
         Plugins.getMemberAccessor().invoke(MySQLIncrementalDumper.class.getDeclaredMethod("handleEvent", AbstractBinlogEvent.class), incrementalDumper, rowsEvent);
-        List<Record> actual = channel.fetchRecords(1, 0);
+        List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
         assertThat(actual.size(), is(1));
         assertThat(actual.get(0), instanceOf(PlaceholderRecord.class));
     }
diff --git a/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java b/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java
index c70036c8853..1f2f673ff7e 100644
--- a/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java
+++ b/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java
@@ -49,6 +49,7 @@ import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Collections;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -133,6 +134,6 @@ class PostgreSQLWALDumperTest {
             walDumper.start();
         } catch (final IngestException ignored) {
         }
-        assertThat(channel.fetchRecords(100, 0).size(), is(1));
+        assertThat(channel.fetchRecords(100, 0, TimeUnit.SECONDS).size(), is(1));
     }
 }
diff --git a/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java b/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java
index 1620b0be8b0..d5466ba5ee3 100644
--- a/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java
+++ b/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java
@@ -93,7 +93,7 @@ class DataSourceImporterTest {
     void assertWriteInsertDataRecord() throws SQLException {
         DataRecord insertRecord = getDataRecord("INSERT");
         when(connection.prepareStatement(any())).thenReturn(preparedStatement);
-        when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(insertRecord));
+        when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(insertRecord));
         jdbcImporter.run();
         verify(preparedStatement).setObject(1, 1);
         verify(preparedStatement).setObject(2, 10);
@@ -105,7 +105,7 @@ class DataSourceImporterTest {
     void assertDeleteDataRecord() throws SQLException {
         DataRecord deleteRecord = getDataRecord("DELETE");
         when(connection.prepareStatement(any())).thenReturn(preparedStatement);
-        when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(deleteRecord));
+        when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(deleteRecord));
         jdbcImporter.run();
         verify(preparedStatement).setObject(1, 1);
         verify(preparedStatement).setObject(2, 10);
@@ -116,7 +116,7 @@ class DataSourceImporterTest {
     void assertUpdateDataRecord() throws SQLException {
         DataRecord updateRecord = getDataRecord("UPDATE");
         when(connection.prepareStatement(any())).thenReturn(preparedStatement);
-        when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(updateRecord));
+        when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(updateRecord));
         jdbcImporter.run();
         verify(preparedStatement).setObject(1, 20);
         verify(preparedStatement).setObject(2, "UPDATE");
@@ -129,7 +129,7 @@ class DataSourceImporterTest {
     void assertUpdatePrimaryKeyDataRecord() throws SQLException {
         DataRecord updateRecord = getUpdatePrimaryKeyDataRecord();
         when(connection.prepareStatement(any())).thenReturn(preparedStatement);
-        when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(updateRecord));
+        when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(updateRecord));
         jdbcImporter.run();
         InOrder inOrder = inOrder(preparedStatement);
         inOrder.verify(preparedStatement).setObject(1, 2);