You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2023/02/16 09:33:38 UTC

[flink-connector-pulsar] 01/05: [FLINK-28959][Stream] Add retry and rate limit for admin request.

This is an automated email from the ASF dual-hosted git repository.

guoweijie pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-pulsar.git

commit 471d24ca266e955adfa5262976e3ab7103ad1d5e
Author: Yufan Sheng <yu...@streamnative.io>
AuthorDate: Thu Feb 16 14:09:05 2023 +0800

    [FLINK-28959][Stream] Add retry and rate limit for admin request.
---
 .../generated/pulsar_admin_configuration.html      |  18 ++
 .../common/config/PulsarAdminProxyBuilder.java     |  64 +++++
 .../pulsar/common/config/PulsarClientFactory.java  |   4 +-
 .../pulsar/common/config/PulsarOptions.java        |  26 ++
 .../handler/PulsarAdminInvocationHandler.java      | 146 ++++++++++
 .../source/reader/PulsarPartitionSplitReader.java  |   9 +-
 .../handler/PulsarAdminInvocationHandlerTest.java  | 309 +++++++++++++++++++++
 .../reader/PulsarPartitionSplitReaderTest.java     |  14 +-
 8 files changed, 579 insertions(+), 11 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html b/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
index 31c7273..fff2f24 100644
--- a/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
+++ b/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
@@ -32,11 +32,29 @@
             <td>Integer</td>
             <td>The server response read timeout (in ms) for the PulsarAdmin client for any request.</td>
         </tr>
+        <tr>
+            <td><h5>pulsar.admin.requestRates</h5></td>
+            <td style="word-wrap: break-word;">5</td>
+            <td>Integer</td>
+            <td>It will add ratelimit for PulsarAdmin metadata requests, stands for requests per second.</td>
+        </tr>
+        <tr>
+            <td><h5>pulsar.admin.requestRetries</h5></td>
+            <td style="word-wrap: break-word;">5</td>
+            <td>Integer</td>
+            <td>For PulsarAdmin request, it will retry until we get a success response, fail if we exhausted retry count.</td>
+        </tr>
         <tr>
             <td><h5>pulsar.admin.requestTimeout</h5></td>
             <td style="word-wrap: break-word;">300000</td>
             <td>Integer</td>
             <td>The server request timeout (in ms) for the PulsarAdmin client for any request.</td>
         </tr>
+        <tr>
+            <td><h5>pulsar.admin.requestWaitMillis</h5></td>
+            <td style="word-wrap: break-word;">3000</td>
+            <td>Long</td>
+            <td>For PulsarAdmin request, We will sleep the given time before retrying the failed request.</td>
+        </tr>
     </tbody>
 </table>
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarAdminProxyBuilder.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarAdminProxyBuilder.java
new file mode 100644
index 0000000..5c8da6b
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarAdminProxyBuilder.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.common.config;
+
+import org.apache.flink.connector.pulsar.common.handler.PulsarAdminInvocationHandler;
+
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl;
+import org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector;
+import org.apache.pulsar.client.api.PulsarClientException;
+
+import java.lang.reflect.Proxy;
+
+/**
+ * {@link org.apache.pulsar.client.admin.PulsarAdminBuilder} didn't expose all the configurations to
+ * end user. We have to extend the default builder method for adding extra configurations.
+ */
+public class PulsarAdminProxyBuilder extends PulsarAdminBuilderImpl {
+
+    private final PulsarConfiguration configuration;
+
+    public PulsarAdminProxyBuilder(PulsarConfiguration configuration) {
+        this.configuration = configuration;
+    }
+
+    /**
+     * This is used by the internal implementation of {@link AsyncHttpConnector} to set the max
+     * allowed number of request threads.
+     */
+    public void numIoThreads(int numIoThreads) {
+        conf.setNumIoThreads(numIoThreads);
+    }
+
+    /**
+     * Wrap the pulsar admin interface into a proxy instance which can retry the request and limit
+     * the request rate.
+     */
+    @Override
+    public PulsarAdmin build() throws PulsarClientException {
+        PulsarAdminInvocationHandler handler =
+                new PulsarAdminInvocationHandler(super.build(), configuration);
+        return (PulsarAdmin)
+                Proxy.newProxyInstance(
+                        PulsarAdmin.class.getClassLoader(),
+                        new Class[] {PulsarAdmin.class},
+                        handler);
+    }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
index 7b529bd..7b45ef1 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
@@ -22,7 +22,6 @@ import org.apache.flink.annotation.Internal;
 import org.apache.flink.configuration.ConfigOption;
 
 import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminBuilder;
 import org.apache.pulsar.client.api.Authentication;
 import org.apache.pulsar.client.api.AuthenticationFactory;
 import org.apache.pulsar.client.api.ClientBuilder;
@@ -193,7 +192,7 @@ public final class PulsarClientFactory {
      */
     public static PulsarAdmin createAdmin(PulsarConfiguration configuration)
             throws PulsarClientException {
-        PulsarAdminBuilder builder = PulsarAdmin.builder();
+        PulsarAdminProxyBuilder builder = new PulsarAdminProxyBuilder(configuration);
 
         // Create the authentication instance for the Pulsar client.
         builder.authentication(createAuthentication(configuration));
@@ -223,6 +222,7 @@ public final class PulsarClientFactory {
                 PULSAR_REQUEST_TIMEOUT, v -> builder.requestTimeout(v, MILLISECONDS));
         configuration.useOption(
                 PULSAR_AUTO_CERT_REFRESH_TIME, v -> builder.autoCertRefreshTime(v, MILLISECONDS));
+        configuration.useOption(PULSAR_NUM_IO_THREADS, builder::numIoThreads);
 
         return builder.build();
     }
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
index c3ebdf4..9fdc1cf 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
@@ -27,6 +27,7 @@ import org.apache.flink.configuration.description.Description;
 
 import org.apache.pulsar.client.api.ProxyProtocol;
 
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -627,4 +628,29 @@ public final class PulsarOptions {
                     .defaultValue(300000)
                     .withDescription(
                             "The auto cert refresh time (in ms) if Pulsar admin supports TLS authentication.");
+
+    // These config options below are passing to PulsarAdminInvocationHandler.
+    // A wrapper for PulsarAdmin.
+
+    public static final ConfigOption<Integer> PULSAR_ADMIN_REQUEST_RETRIES =
+            ConfigOptions.key(ADMIN_CONFIG_PREFIX + "requestRetries")
+                    .intType()
+                    .defaultValue(5)
+                    .withDescription(
+                            "For PulsarAdmin request, it will retry until we get a success response,"
+                                    + " fail if we exhausted retry count.");
+
+    public static final ConfigOption<Long> PULSAR_ADMIN_REQUEST_WAIT_MILLIS =
+            ConfigOptions.key(ADMIN_CONFIG_PREFIX + "requestWaitMillis")
+                    .longType()
+                    .defaultValue(Duration.ofSeconds(3).toMillis())
+                    .withDescription(
+                            "For PulsarAdmin request, We will sleep the given time before retrying the failed request.");
+
+    public static final ConfigOption<Integer> PULSAR_ADMIN_REQUEST_RATES =
+            ConfigOptions.key(ADMIN_CONFIG_PREFIX + "requestRates")
+                    .intType()
+                    .defaultValue(5)
+                    .withDescription(
+                            "It will add ratelimit for PulsarAdmin metadata requests, stands for requests per second.");
 }
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandler.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandler.java
new file mode 100644
index 0000000..b797e2e
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandler.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.common.handler;
+
+import org.apache.flink.connector.pulsar.common.config.PulsarConfiguration;
+
+import org.apache.flink.shaded.guava30.com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RATES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RETRIES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_WAIT_MILLIS;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+/** A wrapper which wraps the {@link PulsarAdmin} with request retry and rate limit support. */
+public class PulsarAdminInvocationHandler implements InvocationHandler {
+    private static final Logger LOG = LoggerFactory.getLogger(PulsarAdminInvocationHandler.class);
+
+    @SuppressWarnings("java:S3077")
+    private static volatile RateLimiter rateLimiter;
+
+    private final PulsarAdmin admin;
+    private final int retryTimes;
+    private final long waitMillis;
+    private final int requestRates;
+    private final Map<String, Object> handlers;
+
+    public PulsarAdminInvocationHandler(PulsarAdmin admin, PulsarConfiguration configuration) {
+        this.admin = admin;
+        this.retryTimes = configuration.get(PULSAR_ADMIN_REQUEST_RETRIES);
+        this.waitMillis = configuration.get(PULSAR_ADMIN_REQUEST_WAIT_MILLIS);
+        this.requestRates = configuration.get(PULSAR_ADMIN_REQUEST_RATES);
+        this.handlers = new ConcurrentHashMap<>();
+    }
+
+    @Override
+    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+        Class<?> returnType = method.getReturnType();
+
+        // No need to proxy the void return type.
+        // The non-interface type is not able to proxy.
+        if (returnType.equals(Void.TYPE) || !returnType.isInterface()) {
+            return method.invoke(admin, args);
+        }
+
+        String methodName = method.getName();
+        if (handlers.containsKey(methodName)) {
+            return handlers.get(methodName);
+        }
+
+        Object handler =
+                Proxy.newProxyInstance(
+                        Thread.currentThread().getContextClassLoader(),
+                        new Class[] {returnType},
+                        new RequestHandler(method.invoke(admin, args)));
+        this.handlers.put(methodName, handler);
+
+        return handler;
+    }
+
+    /** A proxy handler with retry support for all the admin request. */
+    @SuppressWarnings({"java:S1193", "java:S1181", "java:S3776", "java:S112"})
+    private class RequestHandler implements InvocationHandler {
+
+        private final Object handler;
+
+        public RequestHandler(Object handler) {
+            this.handler = handler;
+        }
+
+        @Override
+        public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+            return doInvoke(method, args, retryTimes);
+        }
+
+        private Object doInvoke(Method method, Object[] args, int remainingTimes) throws Throwable {
+            while (true) {
+                // Make sure the request is allowed in the given rates.
+                requestRateLimit(requestRates);
+
+                try {
+                    return method.invoke(handler, args);
+                } catch (InvocationTargetException e) {
+                    Throwable throwable = e.getTargetException();
+                    if (throwable instanceof NotFoundException) {
+                        // No need to retry on such exceptions.
+                        throw throwable;
+                    } else if (throwable instanceof PulsarAdminException) {
+                        remainingTimes--;
+                        LOG.warn("Request error in Admin API, remain times: {}", remainingTimes, e);
+                        if (remainingTimes == 0) {
+                            throw throwable;
+                        } else {
+                            // Sleep for the given times before executing the next query.
+                            sleepUninterruptibly(waitMillis, MILLISECONDS);
+                        }
+                    } else {
+                        throw throwable;
+                    }
+                }
+            }
+        }
+    }
+
+    /** Request a global ratelimit which limits the total admin API request rates. */
+    private static void requestRateLimit(int requestRates) {
+        if (rateLimiter == null) {
+            synchronized (PulsarAdminInvocationHandler.class) {
+                if (rateLimiter == null) {
+                    rateLimiter = RateLimiter.create(requestRates);
+                }
+            }
+        }
+
+        rateLimiter.acquire();
+    }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java
index ca4dbc3..8b1a61f 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java
@@ -95,8 +95,8 @@ public class PulsarPartitionSplitReader
     private static final Logger LOG = LoggerFactory.getLogger(PulsarPartitionSplitReader.class);
 
     private final PulsarClient pulsarClient;
-    @VisibleForTesting final PulsarAdmin pulsarAdmin;
-    @VisibleForTesting final SourceConfiguration sourceConfiguration;
+    private final PulsarAdmin pulsarAdmin;
+    private final SourceConfiguration sourceConfiguration;
     private final Schema<byte[]> schema;
     private final PulsarCrypto pulsarCrypto;
     private final SourceReaderMetricGroup metricGroup;
@@ -354,4 +354,9 @@ public class PulsarPartitionSplitReader
             group.gauge(MSG_NUM_IN_RECEIVER_QUEUE, stats::getMsgNumInReceiverQueue);
         }
     }
+
+    @VisibleForTesting
+    String getSubscriptionName() {
+        return sourceConfiguration.getSubscriptionName();
+    }
 }
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandlerTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandlerTest.java
new file mode 100644
index 0000000..c723a31
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandlerTest.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.common.handler;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
+
+import org.apache.pulsar.client.admin.Bookies;
+import org.apache.pulsar.client.admin.BrokerStats;
+import org.apache.pulsar.client.admin.Brokers;
+import org.apache.pulsar.client.admin.Clusters;
+import org.apache.pulsar.client.admin.Functions;
+import org.apache.pulsar.client.admin.Lookup;
+import org.apache.pulsar.client.admin.Namespaces;
+import org.apache.pulsar.client.admin.NonPersistentTopics;
+import org.apache.pulsar.client.admin.Packages;
+import org.apache.pulsar.client.admin.Properties;
+import org.apache.pulsar.client.admin.ProxyStats;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
+import org.apache.pulsar.client.admin.ResourceGroups;
+import org.apache.pulsar.client.admin.ResourceQuotas;
+import org.apache.pulsar.client.admin.Schemas;
+import org.apache.pulsar.client.admin.Sink;
+import org.apache.pulsar.client.admin.Sinks;
+import org.apache.pulsar.client.admin.Source;
+import org.apache.pulsar.client.admin.Sources;
+import org.apache.pulsar.client.admin.Tenants;
+import org.apache.pulsar.client.admin.TopicPolicies;
+import org.apache.pulsar.client.admin.Topics;
+import org.apache.pulsar.client.admin.Transactions;
+import org.apache.pulsar.client.admin.Worker;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Proxy;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RATES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RETRIES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_WAIT_MILLIS;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/** Unit test of {@link PulsarAdminInvocationHandler}. */
+class PulsarAdminInvocationHandlerTest {
+
+    @Test
+    void retryWithFixedTimesOnRequestFailure() {
+        int retryTimes = 5 + ThreadLocalRandom.current().nextInt(5);
+        String errorMsg = "Always failed with reason: " + randomAlphanumeric(10);
+
+        PulsarAdminTestImpl admin = new PulsarAdminTestImpl(errorMsg);
+
+        Configuration configuration = new Configuration();
+        configuration.set(PULSAR_ADMIN_REQUEST_RETRIES, retryTimes);
+        configuration.set(PULSAR_ADMIN_REQUEST_WAIT_MILLIS, 50L);
+        configuration.set(PULSAR_ADMIN_REQUEST_RATES, 1000);
+        SinkConfiguration sinkConfiguration = new SinkConfiguration(configuration);
+
+        PulsarAdminInvocationHandler handler =
+                new PulsarAdminInvocationHandler(admin, sinkConfiguration);
+
+        PulsarAdmin proxyAdmin =
+                (PulsarAdmin)
+                        Proxy.newProxyInstance(
+                                PulsarAdmin.class.getClassLoader(),
+                                new Class[] {PulsarAdmin.class},
+                                handler);
+
+        assertThatThrownBy(() -> proxyAdmin.lookups().lookupPartitionedTopic("aa"))
+                .isInstanceOf(PulsarAdminException.class)
+                .hasMessage(errorMsg);
+        assertEquals(retryTimes, admin.lookup.callingTimes());
+    }
+
+    @Test
+    void didNotRetryOnNotFoundException() {
+        PulsarAdminTestImpl admin = new PulsarAdminTestImpl("not found");
+
+        PulsarAdminInvocationHandler handler =
+                new PulsarAdminInvocationHandler(admin, new SinkConfiguration(new Configuration()));
+
+        PulsarAdmin proxyAdmin =
+                (PulsarAdmin)
+                        Proxy.newProxyInstance(
+                                PulsarAdmin.class.getClassLoader(),
+                                new Class[] {PulsarAdmin.class},
+                                handler);
+
+        assertThatThrownBy(() -> proxyAdmin.lookups().lookupTopic("some"))
+                .isInstanceOf(NotFoundException.class)
+                .hasMessage("not found");
+        assertEquals(1, admin.lookup.callingTimes());
+    }
+
+    /** Test implementation for PulsarAdmin. */
+    private static final class PulsarAdminTestImpl implements PulsarAdmin {
+
+        private final LookupTestImpl lookup;
+
+        private PulsarAdminTestImpl(String errorMsg) {
+            this.lookup = new LookupTestImpl(errorMsg);
+        }
+
+        @Override
+        public Clusters clusters() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Brokers brokers() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Tenants tenants() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public ResourceGroups resourcegroups() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Properties properties() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Namespaces namespaces() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Topics topics() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public TopicPolicies topicPolicies() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public TopicPolicies topicPolicies(boolean isGlobal) {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Bookies bookies() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public NonPersistentTopics nonPersistentTopics() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public ResourceQuotas resourceQuotas() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Lookup lookups() {
+            lookup.resetTimes();
+            return lookup;
+        }
+
+        @Override
+        public Functions functions() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Source source() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Sources sources() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Sink sink() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Sinks sinks() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Worker worker() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public BrokerStats brokerStats() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public ProxyStats proxyStats() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public String getServiceUrl() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Schemas schemas() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Packages packages() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Transactions transactions() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public void close() {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+    }
+
+    /** Test implementation for Lookup. */
+    private static final class LookupTestImpl implements Lookup {
+
+        private final String errorMsg;
+        private int times;
+
+        private LookupTestImpl(String errorMsg) {
+            this.errorMsg = errorMsg;
+        }
+
+        @Override
+        public String lookupTopic(String topic) throws PulsarAdminException {
+            times++;
+            throw new NotFoundException(
+                    new IllegalArgumentException("not found"), "not found", 404);
+        }
+
+        @Override
+        public CompletableFuture<String> lookupTopicAsync(String topic) {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public Map<String, String> lookupPartitionedTopic(String topic)
+                throws PulsarAdminException {
+            times++;
+            throw new PulsarAdminException(errorMsg);
+        }
+
+        @Override
+        public CompletableFuture<Map<String, String>> lookupPartitionedTopicAsync(String topic) {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public String getBundleRange(String topic) {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        @Override
+        public CompletableFuture<String> getBundleRangeAsync(String topic) {
+            throw new UnsupportedOperationException("We didn't support this method in test.");
+        }
+
+        public int callingTimes() {
+            return times;
+        }
+
+        public void resetTimes() {
+            this.times = 0;
+        }
+    }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java
index 0fdced1..99bdfeb 100644
--- a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java
@@ -30,7 +30,6 @@ import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition;
 import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
 import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
 
-import org.apache.pulsar.client.admin.PulsarAdmin;
 import org.apache.pulsar.client.api.Message;
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -321,16 +320,17 @@ class PulsarPartitionSplitReaderTest extends PulsarTestSuiteBase {
 
         // Create the subscription and set the start position for this reader.
         // Remember not to use Consumer.seek(startPosition)
-        SourceConfiguration sourceConfiguration = reader.sourceConfiguration;
-        PulsarAdmin pulsarAdmin = reader.pulsarAdmin;
-        String subscriptionName = sourceConfiguration.getSubscriptionName();
-        List<String> subscriptions = pulsarAdmin.topics().getSubscriptions(topicName);
+        String subscriptionName = reader.getSubscriptionName();
+        List<String> subscriptions = operator().admin().topics().getSubscriptions(topicName);
         if (!subscriptions.contains(subscriptionName)) {
             // If this subscription is not available. Just create it.
-            pulsarAdmin.topics().createSubscription(topicName, subscriptionName, startPosition);
+            operator()
+                    .admin()
+                    .topics()
+                    .createSubscription(topicName, subscriptionName, startPosition);
         } else {
             // Reset the subscription if this is existed.
-            pulsarAdmin.topics().resetCursor(topicName, subscriptionName, startPosition);
+            operator().admin().topics().resetCursor(topicName, subscriptionName, startPosition);
         }
 
         // Accept the split and start consuming.