You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ro...@apache.org on 2024/06/06 12:18:50 UTC
(flink) branch master updated (ceb4db4d88a -> 9708f9fd657)
This is an automated email from the ASF dual-hosted git repository.
roman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from ceb4db4d88a [FLINK-35532][Runtime/Web Frontend] Prevent Cross-Site Authentication (XSA) attacks on Flink dashboard
new 5855f535498 [FLINK-35501] Replace inheritance with encapsulation for RocksDBStateDataTransfer*
new 9708f9fd657 [FLINK-35501] Use common IO thread pool for RocksDB data transfer
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../generated/expert_rocksdb_section.html | 2 +-
.../generated/rocksdb_configuration.html | 2 +-
.../state/RocksDBKeyedStateBackendBuilder.java | 7 ++-
.../contrib/streaming/state/RocksDBOptions.java | 6 +-
.../streaming/state/RocksDBStateDataTransfer.java | 47 --------------
.../state/RocksDBStateDataTransferHelper.java | 72 ++++++++++++++++++++++
.../streaming/state/RocksDBStateDownloader.java | 22 ++++++-
.../streaming/state/RocksDBStateUploader.java | 20 +++++-
.../RocksDBIncrementalRestoreOperation.java | 11 +++-
.../snapshot/RocksDBSnapshotStrategyBase.java | 2 +-
.../snapshot/RocksIncrementalSnapshotStrategy.java | 3 +-
.../snapshot/RocksNativeFullSnapshotStrategy.java | 3 +-
.../state/EmbeddedRocksDBStateBackendTest.java | 2 +-
13 files changed, 135 insertions(+), 64 deletions(-)
delete mode 100644 flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransfer.java
create mode 100644 flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java
(flink) 02/02: [FLINK-35501] Use common IO thread pool for RocksDB data transfer
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
roman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 9708f9fd65751296b3b0377c964207b630958259
Author: Roman Khachatryan <kh...@gmail.com>
AuthorDate: Wed May 29 22:30:50 2024 +0200
[FLINK-35501] Use common IO thread pool for RocksDB data transfer
Currently, each RocksDB state backend creates an executor backed by a
thread pool.
This makes it difficult to control the total number of threads per TM
because it might have at least one task per slot and theoretically, many
state backends per task (because of chaining).
Additionally, using a common thread pool allows to indirectly control
the load on the underlying DFS (e.g. the total number of requests to S3
from a TM).
This change allows to control the total number of data transfers per TM.
---
docs/layouts/shortcodes/generated/expert_rocksdb_section.html | 2 +-
docs/layouts/shortcodes/generated/rocksdb_configuration.html | 2 +-
.../contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java | 7 ++++---
.../org/apache/flink/contrib/streaming/state/RocksDBOptions.java | 6 +++++-
.../contrib/streaming/state/RocksDBStateDataTransferHelper.java | 9 +++++++++
.../state/restore/RocksDBIncrementalRestoreOperation.java | 9 +++++++--
.../streaming/state/snapshot/RocksDBSnapshotStrategyBase.java | 2 +-
.../state/snapshot/RocksIncrementalSnapshotStrategy.java | 3 ++-
.../state/snapshot/RocksNativeFullSnapshotStrategy.java | 3 ++-
.../contrib/streaming/state/EmbeddedRocksDBStateBackendTest.java | 2 +-
10 files changed, 33 insertions(+), 12 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/expert_rocksdb_section.html b/docs/layouts/shortcodes/generated/expert_rocksdb_section.html
index 47e4edce3ce..eac1d574e26 100644
--- a/docs/layouts/shortcodes/generated/expert_rocksdb_section.html
+++ b/docs/layouts/shortcodes/generated/expert_rocksdb_section.html
@@ -12,7 +12,7 @@
<td><h5>state.backend.rocksdb.checkpoint.transfer.thread.num</h5></td>
<td style="word-wrap: break-word;">4</td>
<td>Integer</td>
- <td>The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend.</td>
+ <td>The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend.If negative, the common (TM) IO thread pool is used (see cluster.io-pool.size)</td>
</tr>
<tr>
<td><h5>state.backend.rocksdb.localdir</h5></td>
diff --git a/docs/layouts/shortcodes/generated/rocksdb_configuration.html b/docs/layouts/shortcodes/generated/rocksdb_configuration.html
index b6ad67234e0..e9cca6e415a 100644
--- a/docs/layouts/shortcodes/generated/rocksdb_configuration.html
+++ b/docs/layouts/shortcodes/generated/rocksdb_configuration.html
@@ -12,7 +12,7 @@
<td><h5>state.backend.rocksdb.checkpoint.transfer.thread.num</h5></td>
<td style="word-wrap: break-word;">4</td>
<td>Integer</td>
- <td>The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend.</td>
+ <td>The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend.If negative, the common (TM) IO thread pool is used (see cluster.io-pool.size)</td>
</tr>
<tr>
<td><h5>state.backend.rocksdb.localdir</h5></td>
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
index e0b1359060b..bcdb32f4e79 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
@@ -536,7 +536,8 @@ public class RocksDBKeyedStateBackendBuilder<K> extends AbstractKeyedStateBacken
overlapFractionThreshold,
useIngestDbRestoreMode,
incrementalRestoreAsyncCompactAfterRescale,
- rescalingUseDeleteFilesInRange);
+ rescalingUseDeleteFilesInRange,
+ ioExecutor);
} else if (priorityQueueConfig.getPriorityQueueStateType()
== EmbeddedRocksDBStateBackend.PriorityQueueStateType.HEAP) {
return new RocksDBHeapTimersFullRestoreOperation<>(
@@ -586,8 +587,8 @@ public class RocksDBKeyedStateBackendBuilder<K> extends AbstractKeyedStateBacken
RocksDBStateUploader stateUploader =
injectRocksDBStateUploader == null
? new RocksDBStateUploader(
- RocksDBStateDataTransferHelper.forThreadNum(
- numberOfTransferingThreads))
+ RocksDBStateDataTransferHelper.forThreadNumIfSpecified(
+ numberOfTransferingThreads, ioExecutor))
: injectRocksDBStateUploader;
if (enableIncrementalCheckpointing) {
checkpointSnapshotStrategy =
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java
index e0fd2303939..e6e0c51a82c 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBOptions.java
@@ -27,6 +27,7 @@ import org.apache.flink.configuration.MemorySize;
import org.apache.flink.configuration.description.Description;
import org.apache.flink.configuration.description.TextElement;
+import static org.apache.flink.configuration.ClusterOptions.CLUSTER_IO_EXECUTOR_POOL_SIZE;
import static org.apache.flink.contrib.streaming.state.EmbeddedRocksDBStateBackend.PriorityQueueStateType.ROCKSDB;
import static org.apache.flink.contrib.streaming.state.PredefinedOptions.DEFAULT;
import static org.apache.flink.contrib.streaming.state.PredefinedOptions.FLASH_SSD_OPTIMIZED;
@@ -86,7 +87,10 @@ public class RocksDBOptions {
.intType()
.defaultValue(4)
.withDescription(
- "The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend.");
+ "The number of threads (per stateful operator) used to transfer (download and upload) files in RocksDBStateBackend."
+ + "If negative, the common (TM) IO thread pool is used (see "
+ + CLUSTER_IO_EXECUTOR_POOL_SIZE.key()
+ + ")");
/** The predefined settings for RocksDB DBOptions and ColumnFamilyOptions by Flink community. */
@Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB)
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java
index 70f31b52616..9e65b8bba91 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java
@@ -38,6 +38,15 @@ public final class RocksDBStateDataTransferHelper implements Closeable {
return new RocksDBStateDataTransferHelper(executorService, executorService::shutdownNow);
}
+ public static RocksDBStateDataTransferHelper forThreadNumIfSpecified(
+ int threadNum, ExecutorService executorService) {
+ if (threadNum >= 0) {
+ return forThreadNum(threadNum);
+ } else {
+ return new RocksDBStateDataTransferHelper(executorService, () -> {});
+ }
+ }
+
private static ExecutorService getExecutorService(int threadNum) {
if (threadNum > 1) {
return Executors.newFixedThreadPool(
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
index 8ee45b372a8..2a9166d0ea0 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
@@ -129,6 +129,8 @@ public class RocksDBIncrementalRestoreOperation<K> implements RocksDBRestoreOper
private final boolean useDeleteFilesInRange;
+ private final ExecutorService ioExecutor;
+
public RocksDBIncrementalRestoreOperation(
String operatorIdentifier,
KeyGroupRange keyGroupRange,
@@ -152,7 +154,8 @@ public class RocksDBIncrementalRestoreOperation<K> implements RocksDBRestoreOper
double overlapFractionThreshold,
boolean useIngestDbRestoreMode,
boolean asyncCompactAfterRescale,
- boolean useDeleteFilesInRange) {
+ boolean useDeleteFilesInRange,
+ ExecutorService ioExecutor) {
this.rocksHandle =
new RocksDBHandle(
kvStateInformation,
@@ -183,6 +186,7 @@ public class RocksDBIncrementalRestoreOperation<K> implements RocksDBRestoreOper
this.useIngestDbRestoreMode = false;
this.asyncCompactAfterRescale = false;
this.useDeleteFilesInRange = useDeleteFilesInRange;
+ this.ioExecutor = ioExecutor;
}
/**
@@ -723,7 +727,8 @@ public class RocksDBIncrementalRestoreOperation<K> implements RocksDBRestoreOper
keyGroupRange.prettyPrintInterval());
try (RocksDBStateDownloader rocksDBStateDownloader =
new RocksDBStateDownloader(
- RocksDBStateDataTransferHelper.forThreadNum(numberOfTransferringThreads))) {
+ RocksDBStateDataTransferHelper.forThreadNumIfSpecified(
+ numberOfTransferringThreads, ioExecutor))) {
rocksDBStateDownloader.transferAllStateDataToDirectory(
downloadSpecs, cancelStreamRegistry);
logger.info(
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java
index 6c3948a6303..f6bb541cf0d 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksDBSnapshotStrategyBase.java
@@ -303,7 +303,7 @@ public abstract class RocksDBSnapshotStrategyBase<K, R extends SnapshotResources
}
@Override
- public abstract void close();
+ public abstract void close() throws IOException;
/** Common operation in native rocksdb snapshot result supplier. */
protected abstract class RocksDBSnapshotOperation
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java
index 48d8bb35aa2..e29518cdd4b 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksIncrementalSnapshotStrategy.java
@@ -47,6 +47,7 @@ import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
@@ -186,7 +187,7 @@ public class RocksIncrementalSnapshotStrategy<K>
}
@Override
- public void close() {
+ public void close() throws IOException {
stateUploader.close();
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksNativeFullSnapshotStrategy.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksNativeFullSnapshotStrategy.java
index 3fc45862dc8..f0b2065635d 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksNativeFullSnapshotStrategy.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/snapshot/RocksNativeFullSnapshotStrategy.java
@@ -44,6 +44,7 @@ import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
@@ -135,7 +136,7 @@ public class RocksNativeFullSnapshotStrategy<K>
}
@Override
- public void close() {
+ public void close() throws IOException {
stateUploader.close();
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/EmbeddedRocksDBStateBackendTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/EmbeddedRocksDBStateBackendTest.java
index d58f388ab44..a8277fa07fc 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/EmbeddedRocksDBStateBackendTest.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/EmbeddedRocksDBStateBackendTest.java
@@ -785,7 +785,7 @@ public class EmbeddedRocksDBStateBackendTest
assertThat(keyedStateBackend.isDisposed()).isTrue();
}
- private void verifyRocksDBStateUploaderClosed() {
+ private void verifyRocksDBStateUploaderClosed() throws IOException {
if (enableIncrementalCheckpointing) {
verify(rocksDBStateUploader, times(1)).close();
}
(flink) 01/02: [FLINK-35501] Replace inheritance with encapsulation for RocksDBStateDataTransfer*
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
roman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 5855f5354985cabea6f01b6b2effaaa8cfcbee55
Author: Roman Khachatryan <kh...@gmail.com>
AuthorDate: Wed May 29 21:46:11 2024 +0200
[FLINK-35501] Replace inheritance with encapsulation for RocksDBStateDataTransfer*
---
.../state/RocksDBKeyedStateBackendBuilder.java | 4 +-
.../streaming/state/RocksDBStateDataTransfer.java | 47 ----------------
.../state/RocksDBStateDataTransferHelper.java | 63 ++++++++++++++++++++++
.../streaming/state/RocksDBStateDownloader.java | 22 ++++++--
.../streaming/state/RocksDBStateUploader.java | 20 +++++--
.../RocksDBIncrementalRestoreOperation.java | 4 +-
6 files changed, 105 insertions(+), 55 deletions(-)
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
index 1a63950305d..e0b1359060b 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackendBuilder.java
@@ -585,7 +585,9 @@ public class RocksDBKeyedStateBackendBuilder<K> extends AbstractKeyedStateBacken
RocksDBSnapshotStrategyBase<K, ?> checkpointSnapshotStrategy;
RocksDBStateUploader stateUploader =
injectRocksDBStateUploader == null
- ? new RocksDBStateUploader(numberOfTransferingThreads)
+ ? new RocksDBStateUploader(
+ RocksDBStateDataTransferHelper.forThreadNum(
+ numberOfTransferingThreads))
: injectRocksDBStateUploader;
if (enableIncrementalCheckpointing) {
checkpointSnapshotStrategy =
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransfer.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransfer.java
deleted file mode 100644
index 246332671ce..00000000000
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransfer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.contrib.streaming.state;
-
-import org.apache.flink.util.concurrent.ExecutorThreadFactory;
-
-import java.io.Closeable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.apache.flink.util.concurrent.Executors.newDirectExecutorService;
-
-/** Data transfer base class for {@link RocksDBKeyedStateBackend}. */
-class RocksDBStateDataTransfer implements Closeable {
-
- protected final ExecutorService executorService;
-
- RocksDBStateDataTransfer(int threadNum) {
- if (threadNum > 1) {
- executorService =
- Executors.newFixedThreadPool(
- threadNum, new ExecutorThreadFactory("Flink-RocksDBStateDataTransfer"));
- } else {
- executorService = newDirectExecutorService();
- }
- }
-
- @Override
- public void close() {
- executorService.shutdownNow();
- }
-}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java
new file mode 100644
index 00000000000..70f31b52616
--- /dev/null
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDataTransferHelper.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.contrib.streaming.state;
+
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.concurrent.Executors.newDirectExecutorService;
+
+/** Data transfer helper for {@link RocksDBKeyedStateBackend}. */
+public final class RocksDBStateDataTransferHelper implements Closeable {
+
+ private final ExecutorService executorService;
+ private final Closeable closeable;
+
+ public static RocksDBStateDataTransferHelper forThreadNum(int threadNum) {
+ ExecutorService executorService = getExecutorService(threadNum);
+ return new RocksDBStateDataTransferHelper(executorService, executorService::shutdownNow);
+ }
+
+ private static ExecutorService getExecutorService(int threadNum) {
+ if (threadNum > 1) {
+ return Executors.newFixedThreadPool(
+ threadNum, new ExecutorThreadFactory("Flink-RocksDBStateDataTransferHelper"));
+ } else {
+ return newDirectExecutorService();
+ }
+ }
+
+ RocksDBStateDataTransferHelper(ExecutorService executorService, Closeable closeable) {
+ this.executorService = checkNotNull(executorService);
+ this.closeable = checkNotNull(closeable);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeable.close();
+ }
+
+ public ExecutorService getExecutorService() {
+ return executorService;
+ }
+}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDownloader.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDownloader.java
index 850bda6bad7..6c319cb9358 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDownloader.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateDownloader.java
@@ -17,6 +17,7 @@
package org.apache.flink.contrib.streaming.state;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.runtime.state.StreamStateHandle;
@@ -27,6 +28,7 @@ import org.apache.flink.util.IOUtils;
import org.apache.flink.util.concurrent.FutureUtils;
import org.apache.flink.util.function.ThrowingRunnable;
+import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
@@ -38,10 +40,16 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/** Help class for downloading RocksDB state files. */
-public class RocksDBStateDownloader extends RocksDBStateDataTransfer {
+public class RocksDBStateDownloader implements Closeable {
+ private final RocksDBStateDataTransferHelper transfer;
+ @VisibleForTesting
public RocksDBStateDownloader(int restoringThreadNum) {
- super(restoringThreadNum);
+ this(RocksDBStateDataTransferHelper.forThreadNum(restoringThreadNum));
+ }
+
+ public RocksDBStateDownloader(RocksDBStateDataTransferHelper transfer) {
+ this.transfer = transfer;
}
/**
@@ -116,7 +124,10 @@ public class RocksDBStateDownloader extends RocksDBStateDataTransfer {
remoteFileHandle,
closeableRegistry));
}))
- .map(runnable -> CompletableFuture.runAsync(runnable, executorService));
+ .map(
+ runnable ->
+ CompletableFuture.runAsync(
+ runnable, transfer.getExecutorService()));
}
/** Copies the file from a single state handle to the given path. */
@@ -155,4 +166,9 @@ public class RocksDBStateDownloader extends RocksDBStateDataTransfer {
throw new IOException(ex);
}
}
+
+ @Override
+ public void close() throws IOException {
+ this.transfer.close();
+ }
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateUploader.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateUploader.java
index 983450fad75..bb89729cef9 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateUploader.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateUploader.java
@@ -18,6 +18,7 @@
package org.apache.flink.contrib.streaming.state;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.runtime.state.CheckpointStateOutputStream;
import org.apache.flink.runtime.state.CheckpointStreamFactory;
@@ -33,6 +34,7 @@ import org.apache.flink.util.function.CheckedSupplier;
import javax.annotation.Nonnull;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
@@ -44,11 +46,18 @@ import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/** Help class for uploading RocksDB state files. */
-public class RocksDBStateUploader extends RocksDBStateDataTransfer {
+public class RocksDBStateUploader implements Closeable {
private static final int READ_BUFFER_SIZE = 16 * 1024;
+ private final RocksDBStateDataTransferHelper transfer;
+
+ @VisibleForTesting
public RocksDBStateUploader(int numberOfSnapshottingThreads) {
- super(numberOfSnapshottingThreads);
+ this(RocksDBStateDataTransferHelper.forThreadNum(numberOfSnapshottingThreads));
+ }
+
+ public RocksDBStateUploader(RocksDBStateDataTransferHelper transfer) {
+ this.transfer = transfer;
}
/**
@@ -114,7 +123,7 @@ public class RocksDBStateUploader extends RocksDBStateDataTransfer {
stateScope,
closeableRegistry,
tmpResourcesRegistry)),
- executorService))
+ transfer.getExecutorService()))
.collect(Collectors.toList());
}
@@ -170,4 +179,9 @@ public class RocksDBStateUploader extends RocksDBStateDataTransfer {
}
}
}
+
+ @Override
+ public void close() throws IOException {
+ this.transfer.close();
+ }
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
index b5276abc848..8ee45b372a8 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java
@@ -24,6 +24,7 @@ import org.apache.flink.contrib.streaming.state.RocksDBIncrementalCheckpointUtil
import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend.RocksDbKvStateInfo;
import org.apache.flink.contrib.streaming.state.RocksDBNativeMetricOptions;
import org.apache.flink.contrib.streaming.state.RocksDBOperationUtils;
+import org.apache.flink.contrib.streaming.state.RocksDBStateDataTransferHelper;
import org.apache.flink.contrib.streaming.state.RocksDBStateDownloader;
import org.apache.flink.contrib.streaming.state.RocksDBWriteBatchWrapper;
import org.apache.flink.contrib.streaming.state.RocksIteratorWrapper;
@@ -721,7 +722,8 @@ public class RocksDBIncrementalRestoreOperation<K> implements RocksDBRestoreOper
operatorIdentifier,
keyGroupRange.prettyPrintInterval());
try (RocksDBStateDownloader rocksDBStateDownloader =
- new RocksDBStateDownloader(numberOfTransferringThreads)) {
+ new RocksDBStateDownloader(
+ RocksDBStateDataTransferHelper.forThreadNum(numberOfTransferringThreads))) {
rocksDBStateDownloader.transferAllStateDataToDirectory(
downloadSpecs, cancelStreamRegistry);
logger.info(