You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by xt...@apache.org on 2023/05/15 02:59:08 UTC

[flink] 01/02: [FLINK-31638][network] Introduce the TieredStorageConsumerClient and TierConsumerAgent

This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 9d75f2447c78b3302988efac75baea8011adeea3
Author: Wencong Liu <li...@163.com>
AuthorDate: Fri May 5 17:06:07 2023 +0800

    [FLINK-31638][network] Introduce the TieredStorageConsumerClient and TierConsumerAgent
---
 .../storage/TieredStorageConsumerClient.java       | 49 +++++++++++++++++++++-
 .../hybrid/tiered/tier/TierConsumerAgent.java      | 22 +++++++++-
 2 files changed, 67 insertions(+), 4 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageConsumerClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageConsumerClient.java
index f1b69c347dc..8f94cc2c825 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageConsumerClient.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/storage/TieredStorageConsumerClient.java
@@ -18,5 +18,50 @@
 
 package org.apache.flink.runtime.io.network.partition.hybrid.tiered.storage;
 
-/** Client of the Tiered Storage used by the consumer. */
-public class TieredStorageConsumerClient {}
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+import org.apache.flink.runtime.io.network.partition.hybrid.tiered.tier.TierConsumerAgent;
+import org.apache.flink.runtime.io.network.partition.hybrid.tiered.tier.TierFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/** {@link TieredStorageConsumerClient} is used to read buffer from tiered store. */
+public class TieredStorageConsumerClient {
+
+    private final List<TierConsumerAgent> tierConsumerAgents;
+
+    public TieredStorageConsumerClient(List<TierFactory> tierFactories) {
+        this.tierConsumerAgents = createTierConsumerAgents(tierFactories);
+    }
+
+    public void start() {
+        for (TierConsumerAgent tierConsumerAgent : tierConsumerAgents) {
+            tierConsumerAgent.start();
+        }
+    }
+
+    public Optional<Buffer> getNextBuffer(int subpartitionId) {
+        // TODO, the detailed logic will be completed when the memory tier is introduced..
+        return Optional.empty();
+    }
+
+    public void close() throws IOException {
+        for (TierConsumerAgent tierConsumerAgent : tierConsumerAgents) {
+            tierConsumerAgent.close();
+        }
+    }
+
+    // --------------------------------------------------------------------------------------------
+    //  Internal methods
+    // --------------------------------------------------------------------------------------------
+
+    private List<TierConsumerAgent> createTierConsumerAgents(List<TierFactory> tierFactories) {
+        ArrayList<TierConsumerAgent> tierConsumerAgents = new ArrayList<>();
+        for (TierFactory tierFactory : tierFactories) {
+            tierConsumerAgents.add(tierFactory.createConsumerAgent());
+        }
+        return tierConsumerAgents;
+    }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/TierConsumerAgent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/TierConsumerAgent.java
index b9dc497089c..9f5cdc1cec0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/TierConsumerAgent.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/hybrid/tiered/tier/TierConsumerAgent.java
@@ -18,5 +18,23 @@
 
 package org.apache.flink.runtime.io.network.partition.hybrid.tiered.tier;
 
-/** The consumer-side agent of a Tier. */
-public interface TierConsumerAgent {}
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+
+import java.io.IOException;
+import java.util.Optional;
+
+/**
+ * The {@link TierConsumerAgent} is the consumer agent of each tier in tiered store, which could
+ * read data from responding tier.
+ */
+public interface TierConsumerAgent {
+
+    /** Start the consumer agent. */
+    void start();
+
+    /** Get buffer from the consumer agent. */
+    Optional<Buffer> getNextBuffer(int subpartitionId, int segmentId);
+
+    /** Close the consumer agent. */
+    void close() throws IOException;
+}