You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by az...@apache.org on 2023/04/25 06:17:54 UTC
[shardingsphere] branch master updated: Reduce CDC fetching records waiting time (#25313)
This is an automated email from the ASF dual-hosted git repository.
azexin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 6ce0fe171af Reduce CDC fetching records waiting time (#25313)
6ce0fe171af is described below
commit 6ce0fe171af3f5829bada693e4eb76f07764eaa4
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Tue Apr 25 14:17:47 2023 +0800
Reduce CDC fetching records waiting time (#25313)
* Refactor PipelineChannel.fetchRecords
* Reduce CDC fetching records waiting time
---
.../data/pipeline/api/ingest/channel/PipelineChannel.java | 6 ++++--
.../data/pipeline/cdc/core/importer/SocketSinkImporter.java | 3 ++-
.../data/pipeline/core/importer/DataSourceImporter.java | 3 ++-
.../ingest/channel/memory/MultiplexMemoryPipelineChannel.java | 5 +++--
.../ingest/channel/memory/SimpleMemoryPipelineChannel.java | 5 +++--
.../channel/memory/MultiplexMemoryPipelineChannelTest.java | 2 +-
.../pipeline/mysql/ingest/MySQLIncrementalDumperTest.java | 11 ++++++-----
.../pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java | 3 ++-
.../data/pipeline/core/importer/DataSourceImporterTest.java | 8 ++++----
9 files changed, 27 insertions(+), 19 deletions(-)
diff --git a/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java b/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java
index deec0dee990..5fa2b8ce353 100644
--- a/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java
+++ b/kernel/data-pipeline/api/src/main/java/org/apache/shardingsphere/data/pipeline/api/ingest/channel/PipelineChannel.java
@@ -20,6 +20,7 @@ package org.apache.shardingsphere.data.pipeline.api.ingest.channel;
import org.apache.shardingsphere.data.pipeline.api.ingest.record.Record;
import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* Pipeline channel.
@@ -38,10 +39,11 @@ public interface PipelineChannel {
* It might be blocked at most timeout seconds if available records count doesn't reach batch size.
*
* @param batchSize record batch size
- * @param timeoutSeconds timeout(seconds)
+ * @param timeout timeout
+ * @param timeUnit time unit
* @return record
*/
- List<Record> fetchRecords(int batchSize, int timeoutSeconds);
+ List<Record> fetchRecords(int batchSize, int timeout, TimeUnit timeUnit);
/**
* Ack the last batch.
diff --git a/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java b/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java
index 2fe971b7818..e6dbc6e4d36 100644
--- a/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java
+++ b/kernel/data-pipeline/cdc/core/src/main/java/org/apache/shardingsphere/data/pipeline/cdc/core/importer/SocketSinkImporter.java
@@ -40,6 +40,7 @@ import org.apache.shardingsphere.data.pipeline.spi.ratelimit.JobRateLimitAlgorit
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
@@ -79,7 +80,7 @@ public final class SocketSinkImporter extends AbstractLifecycleExecutor implemen
importerConnector.sendIncrementalStartEvent(this, batchSize);
}
while (isRunning()) {
- List<Record> records = channel.fetchRecords(batchSize, 3);
+ List<Record> records = channel.fetchRecords(batchSize, 500, TimeUnit.MILLISECONDS);
if (null != records && !records.isEmpty()) {
List<Record> recordList = records.stream().filter(each -> !(each instanceof PlaceholderRecord)).collect(Collectors.toList());
try {
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java
index 4cc79efb8dd..83692765b88 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/importer/DataSourceImporter.java
@@ -50,6 +50,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
@@ -93,7 +94,7 @@ public final class DataSourceImporter extends AbstractLifecycleExecutor implemen
protected void runBlocking() {
int batchSize = importerConfig.getBatchSize() * 2;
while (isRunning()) {
- List<Record> records = channel.fetchRecords(batchSize, 3);
+ List<Record> records = channel.fetchRecords(batchSize, 3, TimeUnit.SECONDS);
if (null != records && !records.isEmpty()) {
PipelineJobProgressUpdatedParameter updatedParam = flush(dataSourceManager.getDataSource(importerConfig.getDataSourceConfig()), records);
channel.ack(records);
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java
index a9e3cd49514..aa459ecd92f 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannel.java
@@ -27,6 +27,7 @@ import org.apache.shardingsphere.data.pipeline.api.ingest.record.Record;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -67,8 +68,8 @@ public final class MultiplexMemoryPipelineChannel implements PipelineChannel {
}
@Override
- public List<Record> fetchRecords(final int batchSize, final int timeoutSeconds) {
- return findChannel().fetchRecords(batchSize, timeoutSeconds);
+ public List<Record> fetchRecords(final int batchSize, final int timeout, final TimeUnit timeUnit) {
+ return findChannel().fetchRecords(batchSize, timeout, timeUnit);
}
@Override
diff --git a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java
index 6484aeb0b1f..e292f57d0e7 100644
--- a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java
+++ b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/SimpleMemoryPipelineChannel.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
/**
* Simple memory pipeline channel.
@@ -53,11 +54,11 @@ public final class SimpleMemoryPipelineChannel implements PipelineChannel {
@SneakyThrows(InterruptedException.class)
// TODO thread-safe?
@Override
- public List<Record> fetchRecords(final int batchSize, final int timeoutSeconds) {
+ public List<Record> fetchRecords(final int batchSize, final int timeout, final TimeUnit timeUnit) {
List<Record> result = new ArrayList<>(batchSize);
long start = System.currentTimeMillis();
while (batchSize > queue.size()) {
- if (timeoutSeconds * 1000L <= System.currentTimeMillis() - start) {
+ if (timeUnit.toMillis(timeout) <= System.currentTimeMillis() - start) {
break;
}
Thread.sleep(100L);
diff --git a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java
index 1e99ca65fea..2ca5d558b16 100644
--- a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java
+++ b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/ingest/channel/memory/MultiplexMemoryPipelineChannelTest.java
@@ -87,7 +87,7 @@ class MultiplexMemoryPipelineChannelTest {
private static void fetch(final MultiplexMemoryPipelineChannel memoryChannel, final CountDownLatch countDownLatch) {
int maxLoopCount = 10;
for (int j = 1; j <= maxLoopCount; j++) {
- List<Record> records = memoryChannel.fetchRecords(100, 1);
+ List<Record> records = memoryChannel.fetchRecords(100, 1, TimeUnit.SECONDS);
memoryChannel.ack(records);
records.forEach(each -> countDownLatch.countDown());
if (!records.isEmpty() && records.get(records.size() - 1) instanceof FinishedRecord) {
diff --git a/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java b/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java
index c5f7ac75567..f6653c39c09 100644
--- a/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java
+++ b/kernel/data-pipeline/dialect/mysql/src/test/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLIncrementalDumperTest.java
@@ -63,6 +63,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -150,7 +151,7 @@ class MySQLIncrementalDumperTest {
rowsEvent.setAfterRows(Collections.singletonList(new Serializable[]{101, 1, "OK"}));
Method method = MySQLIncrementalDumper.class.getDeclaredMethod("handleWriteRowsEvent", WriteRowsEvent.class, PipelineTableMetaData.class);
Plugins.getMemberAccessor().invoke(method, incrementalDumper, rowsEvent, pipelineTableMetaData);
- List<Record> actual = channel.fetchRecords(1, 0);
+ List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
assertThat(actual.size(), is(1));
assertThat(actual.get(0), instanceOf(DataRecord.class));
assertThat(((DataRecord) actual.get(0)).getType(), is(IngestDataChangeType.INSERT));
@@ -180,7 +181,7 @@ class MySQLIncrementalDumperTest {
rowsEvent.setAfterRows(Collections.singletonList(new Serializable[]{101, 1, "OK2"}));
Method method = MySQLIncrementalDumper.class.getDeclaredMethod("handleUpdateRowsEvent", UpdateRowsEvent.class, PipelineTableMetaData.class);
Plugins.getMemberAccessor().invoke(method, incrementalDumper, rowsEvent, pipelineTableMetaData);
- List<Record> actual = channel.fetchRecords(1, 0);
+ List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
assertThat(actual.size(), is(1));
assertThat(actual.get(0), instanceOf(DataRecord.class));
assertThat(((DataRecord) actual.get(0)).getType(), is(IngestDataChangeType.UPDATE));
@@ -205,7 +206,7 @@ class MySQLIncrementalDumperTest {
rowsEvent.setBeforeRows(Collections.singletonList(new Serializable[]{101, 1, "OK"}));
Method method = MySQLIncrementalDumper.class.getDeclaredMethod("handleDeleteRowsEvent", DeleteRowsEvent.class, PipelineTableMetaData.class);
Plugins.getMemberAccessor().invoke(method, incrementalDumper, rowsEvent, pipelineTableMetaData);
- List<Record> actual = channel.fetchRecords(1, 0);
+ List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
assertThat(actual.size(), is(1));
assertThat(actual.get(0), instanceOf(DataRecord.class));
assertThat(((DataRecord) actual.get(0)).getType(), is(IngestDataChangeType.DELETE));
@@ -215,7 +216,7 @@ class MySQLIncrementalDumperTest {
@Test
void assertPlaceholderEvent() throws ReflectiveOperationException {
Plugins.getMemberAccessor().invoke(MySQLIncrementalDumper.class.getDeclaredMethod("handleEvent", AbstractBinlogEvent.class), incrementalDumper, new PlaceholderEvent());
- List<Record> actual = channel.fetchRecords(1, 0);
+ List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
assertThat(actual.size(), is(1));
assertThat(actual.get(0), instanceOf(PlaceholderRecord.class));
}
@@ -225,7 +226,7 @@ class MySQLIncrementalDumperTest {
WriteRowsEvent rowsEvent = new WriteRowsEvent();
rowsEvent.setDatabaseName("unknown_database");
Plugins.getMemberAccessor().invoke(MySQLIncrementalDumper.class.getDeclaredMethod("handleEvent", AbstractBinlogEvent.class), incrementalDumper, rowsEvent);
- List<Record> actual = channel.fetchRecords(1, 0);
+ List<Record> actual = channel.fetchRecords(1, 0, TimeUnit.SECONDS);
assertThat(actual.size(), is(1));
assertThat(actual.get(0), instanceOf(PlaceholderRecord.class));
}
diff --git a/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java b/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java
index c70036c8853..1f2f673ff7e 100644
--- a/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java
+++ b/kernel/data-pipeline/dialect/postgresql/src/test/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/PostgreSQLWALDumperTest.java
@@ -49,6 +49,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Collections;
+import java.util.concurrent.TimeUnit;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -133,6 +134,6 @@ class PostgreSQLWALDumperTest {
walDumper.start();
} catch (final IngestException ignored) {
}
- assertThat(channel.fetchRecords(100, 0).size(), is(1));
+ assertThat(channel.fetchRecords(100, 0, TimeUnit.SECONDS).size(), is(1));
}
}
diff --git a/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java b/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java
index 1620b0be8b0..d5466ba5ee3 100644
--- a/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java
+++ b/test/it/pipeline/src/test/java/org/apache/shardingsphere/test/it/data/pipeline/core/importer/DataSourceImporterTest.java
@@ -93,7 +93,7 @@ class DataSourceImporterTest {
void assertWriteInsertDataRecord() throws SQLException {
DataRecord insertRecord = getDataRecord("INSERT");
when(connection.prepareStatement(any())).thenReturn(preparedStatement);
- when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(insertRecord));
+ when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(insertRecord));
jdbcImporter.run();
verify(preparedStatement).setObject(1, 1);
verify(preparedStatement).setObject(2, 10);
@@ -105,7 +105,7 @@ class DataSourceImporterTest {
void assertDeleteDataRecord() throws SQLException {
DataRecord deleteRecord = getDataRecord("DELETE");
when(connection.prepareStatement(any())).thenReturn(preparedStatement);
- when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(deleteRecord));
+ when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(deleteRecord));
jdbcImporter.run();
verify(preparedStatement).setObject(1, 1);
verify(preparedStatement).setObject(2, 10);
@@ -116,7 +116,7 @@ class DataSourceImporterTest {
void assertUpdateDataRecord() throws SQLException {
DataRecord updateRecord = getDataRecord("UPDATE");
when(connection.prepareStatement(any())).thenReturn(preparedStatement);
- when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(updateRecord));
+ when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(updateRecord));
jdbcImporter.run();
verify(preparedStatement).setObject(1, 20);
verify(preparedStatement).setObject(2, "UPDATE");
@@ -129,7 +129,7 @@ class DataSourceImporterTest {
void assertUpdatePrimaryKeyDataRecord() throws SQLException {
DataRecord updateRecord = getUpdatePrimaryKeyDataRecord();
when(connection.prepareStatement(any())).thenReturn(preparedStatement);
- when(channel.fetchRecords(anyInt(), anyInt())).thenReturn(mockRecords(updateRecord));
+ when(channel.fetchRecords(anyInt(), anyInt(), any())).thenReturn(mockRecords(updateRecord));
jdbcImporter.run();
InOrder inOrder = inOrder(preparedStatement);
inOrder.verify(preparedStatement).setObject(1, 2);