You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2023/02/16 09:33:38 UTC
[flink-connector-pulsar] 01/05: [FLINK-28959][Stream] Add retry and rate limit for admin request.
This is an automated email from the ASF dual-hosted git repository.
guoweijie pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-pulsar.git
commit 471d24ca266e955adfa5262976e3ab7103ad1d5e
Author: Yufan Sheng <yu...@streamnative.io>
AuthorDate: Thu Feb 16 14:09:05 2023 +0800
[FLINK-28959][Stream] Add retry and rate limit for admin request.
---
.../generated/pulsar_admin_configuration.html | 18 ++
.../common/config/PulsarAdminProxyBuilder.java | 64 +++++
.../pulsar/common/config/PulsarClientFactory.java | 4 +-
.../pulsar/common/config/PulsarOptions.java | 26 ++
.../handler/PulsarAdminInvocationHandler.java | 146 ++++++++++
.../source/reader/PulsarPartitionSplitReader.java | 9 +-
.../handler/PulsarAdminInvocationHandlerTest.java | 309 +++++++++++++++++++++
.../reader/PulsarPartitionSplitReaderTest.java | 14 +-
8 files changed, 579 insertions(+), 11 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html b/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
index 31c7273..fff2f24 100644
--- a/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
+++ b/docs/layouts/shortcodes/generated/pulsar_admin_configuration.html
@@ -32,11 +32,29 @@
<td>Integer</td>
<td>The server response read timeout (in ms) for the PulsarAdmin client for any request.</td>
</tr>
+ <tr>
+ <td><h5>pulsar.admin.requestRates</h5></td>
+ <td style="word-wrap: break-word;">5</td>
+ <td>Integer</td>
+ <td>It will add ratelimit for PulsarAdmin metadata requests, stands for requests per second.</td>
+ </tr>
+ <tr>
+ <td><h5>pulsar.admin.requestRetries</h5></td>
+ <td style="word-wrap: break-word;">5</td>
+ <td>Integer</td>
+ <td>For PulsarAdmin request, it will retry until we get a success response, fail if we exhausted retry count.</td>
+ </tr>
<tr>
<td><h5>pulsar.admin.requestTimeout</h5></td>
<td style="word-wrap: break-word;">300000</td>
<td>Integer</td>
<td>The server request timeout (in ms) for the PulsarAdmin client for any request.</td>
</tr>
+ <tr>
+ <td><h5>pulsar.admin.requestWaitMillis</h5></td>
+ <td style="word-wrap: break-word;">3000</td>
+ <td>Long</td>
+ <td>For PulsarAdmin request, We will sleep the given time before retrying the failed request.</td>
+ </tr>
</tbody>
</table>
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarAdminProxyBuilder.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarAdminProxyBuilder.java
new file mode 100644
index 0000000..5c8da6b
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarAdminProxyBuilder.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.common.config;
+
+import org.apache.flink.connector.pulsar.common.handler.PulsarAdminInvocationHandler;
+
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.internal.PulsarAdminBuilderImpl;
+import org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector;
+import org.apache.pulsar.client.api.PulsarClientException;
+
+import java.lang.reflect.Proxy;
+
+/**
+ * {@link org.apache.pulsar.client.admin.PulsarAdminBuilder} didn't expose all the configurations to
+ * end user. We have to extend the default builder method for adding extra configurations.
+ */
+public class PulsarAdminProxyBuilder extends PulsarAdminBuilderImpl {
+
+ private final PulsarConfiguration configuration;
+
+ public PulsarAdminProxyBuilder(PulsarConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * This is used by the internal implementation of {@link AsyncHttpConnector} to set the max
+ * allowed number of request threads.
+ */
+ public void numIoThreads(int numIoThreads) {
+ conf.setNumIoThreads(numIoThreads);
+ }
+
+ /**
+ * Wrap the pulsar admin interface into a proxy instance which can retry the request and limit
+ * the request rate.
+ */
+ @Override
+ public PulsarAdmin build() throws PulsarClientException {
+ PulsarAdminInvocationHandler handler =
+ new PulsarAdminInvocationHandler(super.build(), configuration);
+ return (PulsarAdmin)
+ Proxy.newProxyInstance(
+ PulsarAdmin.class.getClassLoader(),
+ new Class[] {PulsarAdmin.class},
+ handler);
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
index 7b529bd..7b45ef1 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarClientFactory.java
@@ -22,7 +22,6 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.configuration.ConfigOption;
import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminBuilder;
import org.apache.pulsar.client.api.Authentication;
import org.apache.pulsar.client.api.AuthenticationFactory;
import org.apache.pulsar.client.api.ClientBuilder;
@@ -193,7 +192,7 @@ public final class PulsarClientFactory {
*/
public static PulsarAdmin createAdmin(PulsarConfiguration configuration)
throws PulsarClientException {
- PulsarAdminBuilder builder = PulsarAdmin.builder();
+ PulsarAdminProxyBuilder builder = new PulsarAdminProxyBuilder(configuration);
// Create the authentication instance for the Pulsar client.
builder.authentication(createAuthentication(configuration));
@@ -223,6 +222,7 @@ public final class PulsarClientFactory {
PULSAR_REQUEST_TIMEOUT, v -> builder.requestTimeout(v, MILLISECONDS));
configuration.useOption(
PULSAR_AUTO_CERT_REFRESH_TIME, v -> builder.autoCertRefreshTime(v, MILLISECONDS));
+ configuration.useOption(PULSAR_NUM_IO_THREADS, builder::numIoThreads);
return builder.build();
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
index c3ebdf4..9fdc1cf 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/config/PulsarOptions.java
@@ -27,6 +27,7 @@ import org.apache.flink.configuration.description.Description;
import org.apache.pulsar.client.api.ProxyProtocol;
+import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -627,4 +628,29 @@ public final class PulsarOptions {
.defaultValue(300000)
.withDescription(
"The auto cert refresh time (in ms) if Pulsar admin supports TLS authentication.");
+
+ // These config options below are passing to PulsarAdminInvocationHandler.
+ // A wrapper for PulsarAdmin.
+
+ public static final ConfigOption<Integer> PULSAR_ADMIN_REQUEST_RETRIES =
+ ConfigOptions.key(ADMIN_CONFIG_PREFIX + "requestRetries")
+ .intType()
+ .defaultValue(5)
+ .withDescription(
+ "For PulsarAdmin request, it will retry until we get a success response,"
+ + " fail if we exhausted retry count.");
+
+ public static final ConfigOption<Long> PULSAR_ADMIN_REQUEST_WAIT_MILLIS =
+ ConfigOptions.key(ADMIN_CONFIG_PREFIX + "requestWaitMillis")
+ .longType()
+ .defaultValue(Duration.ofSeconds(3).toMillis())
+ .withDescription(
+ "For PulsarAdmin request, We will sleep the given time before retrying the failed request.");
+
+ public static final ConfigOption<Integer> PULSAR_ADMIN_REQUEST_RATES =
+ ConfigOptions.key(ADMIN_CONFIG_PREFIX + "requestRates")
+ .intType()
+ .defaultValue(5)
+ .withDescription(
+ "It will add ratelimit for PulsarAdmin metadata requests, stands for requests per second.");
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandler.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandler.java
new file mode 100644
index 0000000..b797e2e
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandler.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.common.handler;
+
+import org.apache.flink.connector.pulsar.common.config.PulsarConfiguration;
+
+import org.apache.flink.shaded.guava30.com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RATES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RETRIES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_WAIT_MILLIS;
+import static org.apache.flink.shaded.guava30.com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+/** A wrapper which wraps the {@link PulsarAdmin} with request retry and rate limit support. */
+public class PulsarAdminInvocationHandler implements InvocationHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(PulsarAdminInvocationHandler.class);
+
+ @SuppressWarnings("java:S3077")
+ private static volatile RateLimiter rateLimiter;
+
+ private final PulsarAdmin admin;
+ private final int retryTimes;
+ private final long waitMillis;
+ private final int requestRates;
+ private final Map<String, Object> handlers;
+
+ public PulsarAdminInvocationHandler(PulsarAdmin admin, PulsarConfiguration configuration) {
+ this.admin = admin;
+ this.retryTimes = configuration.get(PULSAR_ADMIN_REQUEST_RETRIES);
+ this.waitMillis = configuration.get(PULSAR_ADMIN_REQUEST_WAIT_MILLIS);
+ this.requestRates = configuration.get(PULSAR_ADMIN_REQUEST_RATES);
+ this.handlers = new ConcurrentHashMap<>();
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ Class<?> returnType = method.getReturnType();
+
+ // No need to proxy the void return type.
+ // The non-interface type is not able to proxy.
+ if (returnType.equals(Void.TYPE) || !returnType.isInterface()) {
+ return method.invoke(admin, args);
+ }
+
+ String methodName = method.getName();
+ if (handlers.containsKey(methodName)) {
+ return handlers.get(methodName);
+ }
+
+ Object handler =
+ Proxy.newProxyInstance(
+ Thread.currentThread().getContextClassLoader(),
+ new Class[] {returnType},
+ new RequestHandler(method.invoke(admin, args)));
+ this.handlers.put(methodName, handler);
+
+ return handler;
+ }
+
+ /** A proxy handler with retry support for all the admin request. */
+ @SuppressWarnings({"java:S1193", "java:S1181", "java:S3776", "java:S112"})
+ private class RequestHandler implements InvocationHandler {
+
+ private final Object handler;
+
+ public RequestHandler(Object handler) {
+ this.handler = handler;
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ return doInvoke(method, args, retryTimes);
+ }
+
+ private Object doInvoke(Method method, Object[] args, int remainingTimes) throws Throwable {
+ while (true) {
+ // Make sure the request is allowed in the given rates.
+ requestRateLimit(requestRates);
+
+ try {
+ return method.invoke(handler, args);
+ } catch (InvocationTargetException e) {
+ Throwable throwable = e.getTargetException();
+ if (throwable instanceof NotFoundException) {
+ // No need to retry on such exceptions.
+ throw throwable;
+ } else if (throwable instanceof PulsarAdminException) {
+ remainingTimes--;
+ LOG.warn("Request error in Admin API, remain times: {}", remainingTimes, e);
+ if (remainingTimes == 0) {
+ throw throwable;
+ } else {
+ // Sleep for the given times before executing the next query.
+ sleepUninterruptibly(waitMillis, MILLISECONDS);
+ }
+ } else {
+ throw throwable;
+ }
+ }
+ }
+ }
+ }
+
+ /** Request a global ratelimit which limits the total admin API request rates. */
+ private static void requestRateLimit(int requestRates) {
+ if (rateLimiter == null) {
+ synchronized (PulsarAdminInvocationHandler.class) {
+ if (rateLimiter == null) {
+ rateLimiter = RateLimiter.create(requestRates);
+ }
+ }
+ }
+
+ rateLimiter.acquire();
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java
index ca4dbc3..8b1a61f 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReader.java
@@ -95,8 +95,8 @@ public class PulsarPartitionSplitReader
private static final Logger LOG = LoggerFactory.getLogger(PulsarPartitionSplitReader.class);
private final PulsarClient pulsarClient;
- @VisibleForTesting final PulsarAdmin pulsarAdmin;
- @VisibleForTesting final SourceConfiguration sourceConfiguration;
+ private final PulsarAdmin pulsarAdmin;
+ private final SourceConfiguration sourceConfiguration;
private final Schema<byte[]> schema;
private final PulsarCrypto pulsarCrypto;
private final SourceReaderMetricGroup metricGroup;
@@ -354,4 +354,9 @@ public class PulsarPartitionSplitReader
group.gauge(MSG_NUM_IN_RECEIVER_QUEUE, stats::getMsgNumInReceiverQueue);
}
}
+
+ @VisibleForTesting
+ String getSubscriptionName() {
+ return sourceConfiguration.getSubscriptionName();
+ }
}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandlerTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandlerTest.java
new file mode 100644
index 0000000..c723a31
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/common/handler/PulsarAdminInvocationHandlerTest.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.common.handler;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
+
+import org.apache.pulsar.client.admin.Bookies;
+import org.apache.pulsar.client.admin.BrokerStats;
+import org.apache.pulsar.client.admin.Brokers;
+import org.apache.pulsar.client.admin.Clusters;
+import org.apache.pulsar.client.admin.Functions;
+import org.apache.pulsar.client.admin.Lookup;
+import org.apache.pulsar.client.admin.Namespaces;
+import org.apache.pulsar.client.admin.NonPersistentTopics;
+import org.apache.pulsar.client.admin.Packages;
+import org.apache.pulsar.client.admin.Properties;
+import org.apache.pulsar.client.admin.ProxyStats;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.admin.PulsarAdminException;
+import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
+import org.apache.pulsar.client.admin.ResourceGroups;
+import org.apache.pulsar.client.admin.ResourceQuotas;
+import org.apache.pulsar.client.admin.Schemas;
+import org.apache.pulsar.client.admin.Sink;
+import org.apache.pulsar.client.admin.Sinks;
+import org.apache.pulsar.client.admin.Source;
+import org.apache.pulsar.client.admin.Sources;
+import org.apache.pulsar.client.admin.Tenants;
+import org.apache.pulsar.client.admin.TopicPolicies;
+import org.apache.pulsar.client.admin.Topics;
+import org.apache.pulsar.client.admin.Transactions;
+import org.apache.pulsar.client.admin.Worker;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Proxy;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RATES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_RETRIES;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_REQUEST_WAIT_MILLIS;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/** Unit test of {@link PulsarAdminInvocationHandler}. */
+class PulsarAdminInvocationHandlerTest {
+
+ @Test
+ void retryWithFixedTimesOnRequestFailure() {
+ int retryTimes = 5 + ThreadLocalRandom.current().nextInt(5);
+ String errorMsg = "Always failed with reason: " + randomAlphanumeric(10);
+
+ PulsarAdminTestImpl admin = new PulsarAdminTestImpl(errorMsg);
+
+ Configuration configuration = new Configuration();
+ configuration.set(PULSAR_ADMIN_REQUEST_RETRIES, retryTimes);
+ configuration.set(PULSAR_ADMIN_REQUEST_WAIT_MILLIS, 50L);
+ configuration.set(PULSAR_ADMIN_REQUEST_RATES, 1000);
+ SinkConfiguration sinkConfiguration = new SinkConfiguration(configuration);
+
+ PulsarAdminInvocationHandler handler =
+ new PulsarAdminInvocationHandler(admin, sinkConfiguration);
+
+ PulsarAdmin proxyAdmin =
+ (PulsarAdmin)
+ Proxy.newProxyInstance(
+ PulsarAdmin.class.getClassLoader(),
+ new Class[] {PulsarAdmin.class},
+ handler);
+
+ assertThatThrownBy(() -> proxyAdmin.lookups().lookupPartitionedTopic("aa"))
+ .isInstanceOf(PulsarAdminException.class)
+ .hasMessage(errorMsg);
+ assertEquals(retryTimes, admin.lookup.callingTimes());
+ }
+
+ @Test
+ void didNotRetryOnNotFoundException() {
+ PulsarAdminTestImpl admin = new PulsarAdminTestImpl("not found");
+
+ PulsarAdminInvocationHandler handler =
+ new PulsarAdminInvocationHandler(admin, new SinkConfiguration(new Configuration()));
+
+ PulsarAdmin proxyAdmin =
+ (PulsarAdmin)
+ Proxy.newProxyInstance(
+ PulsarAdmin.class.getClassLoader(),
+ new Class[] {PulsarAdmin.class},
+ handler);
+
+ assertThatThrownBy(() -> proxyAdmin.lookups().lookupTopic("some"))
+ .isInstanceOf(NotFoundException.class)
+ .hasMessage("not found");
+ assertEquals(1, admin.lookup.callingTimes());
+ }
+
+ /** Test implementation for PulsarAdmin. */
+ private static final class PulsarAdminTestImpl implements PulsarAdmin {
+
+ private final LookupTestImpl lookup;
+
+ private PulsarAdminTestImpl(String errorMsg) {
+ this.lookup = new LookupTestImpl(errorMsg);
+ }
+
+ @Override
+ public Clusters clusters() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Brokers brokers() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Tenants tenants() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public ResourceGroups resourcegroups() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Properties properties() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Namespaces namespaces() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Topics topics() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public TopicPolicies topicPolicies() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public TopicPolicies topicPolicies(boolean isGlobal) {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Bookies bookies() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public NonPersistentTopics nonPersistentTopics() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public ResourceQuotas resourceQuotas() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Lookup lookups() {
+ lookup.resetTimes();
+ return lookup;
+ }
+
+ @Override
+ public Functions functions() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Source source() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Sources sources() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Sink sink() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Sinks sinks() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Worker worker() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public BrokerStats brokerStats() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public ProxyStats proxyStats() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public String getServiceUrl() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Schemas schemas() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Packages packages() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Transactions transactions() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public void close() {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+ }
+
+ /** Test implementation for Lookup. */
+ private static final class LookupTestImpl implements Lookup {
+
+ private final String errorMsg;
+ private int times;
+
+ private LookupTestImpl(String errorMsg) {
+ this.errorMsg = errorMsg;
+ }
+
+ @Override
+ public String lookupTopic(String topic) throws PulsarAdminException {
+ times++;
+ throw new NotFoundException(
+ new IllegalArgumentException("not found"), "not found", 404);
+ }
+
+ @Override
+ public CompletableFuture<String> lookupTopicAsync(String topic) {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public Map<String, String> lookupPartitionedTopic(String topic)
+ throws PulsarAdminException {
+ times++;
+ throw new PulsarAdminException(errorMsg);
+ }
+
+ @Override
+ public CompletableFuture<Map<String, String>> lookupPartitionedTopicAsync(String topic) {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public String getBundleRange(String topic) {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ @Override
+ public CompletableFuture<String> getBundleRangeAsync(String topic) {
+ throw new UnsupportedOperationException("We didn't support this method in test.");
+ }
+
+ public int callingTimes() {
+ return times;
+ }
+
+ public void resetTimes() {
+ this.times = 0;
+ }
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java
index 0fdced1..99bdfeb 100644
--- a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/source/reader/PulsarPartitionSplitReaderTest.java
@@ -30,7 +30,6 @@ import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition;
import org.apache.flink.connector.pulsar.source.split.PulsarPartitionSplit;
import org.apache.flink.connector.pulsar.testutils.PulsarTestSuiteBase;
-import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.impl.MessageIdImpl;
@@ -321,16 +320,17 @@ class PulsarPartitionSplitReaderTest extends PulsarTestSuiteBase {
// Create the subscription and set the start position for this reader.
// Remember not to use Consumer.seek(startPosition)
- SourceConfiguration sourceConfiguration = reader.sourceConfiguration;
- PulsarAdmin pulsarAdmin = reader.pulsarAdmin;
- String subscriptionName = sourceConfiguration.getSubscriptionName();
- List<String> subscriptions = pulsarAdmin.topics().getSubscriptions(topicName);
+ String subscriptionName = reader.getSubscriptionName();
+ List<String> subscriptions = operator().admin().topics().getSubscriptions(topicName);
if (!subscriptions.contains(subscriptionName)) {
// If this subscription is not available. Just create it.
- pulsarAdmin.topics().createSubscription(topicName, subscriptionName, startPosition);
+ operator()
+ .admin()
+ .topics()
+ .createSubscription(topicName, subscriptionName, startPosition);
} else {
// Reset the subscription if this is existed.
- pulsarAdmin.topics().resetCursor(topicName, subscriptionName, startPosition);
+ operator().admin().topics().resetCursor(topicName, subscriptionName, startPosition);
}
// Accept the split and start consuming.