You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2022/09/02 06:35:05 UTC
[GitHub] [pulsar] kong62 commented on issue #12628: Could not complete the operation. Number of retries has been exhausted. Failed reason: Maximum redirect reached: 5
kong62 commented on issue #12628:
URL: https://github.com/apache/pulsar/issues/12628#issuecomment-1235123147
```
# kubectl logs -f -n pulsar pulsar-broker-0
2022-08-24T06:55:18,640+0000 [AsyncHttpClient-52-1] WARN org.apache.pulsar.client.admin.internal.BaseResource - [http://pulsar-broker-0.pulsar-broker.pulsar.svc.cluster.local:8080/admin/v2/persistent/public/functions/assignments] Failed to perform http put request: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
2022-08-24T06:55:18,640+0000 [main] ERROR org.apache.pulsar.functions.worker.PulsarWorkerService - Error Starting up in worker
org.apache.pulsar.client.admin.PulsarAdminException: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
at org.apache.pulsar.client.admin.internal.BaseResource.getApiException(BaseResource.java:247) ~[org.apache.pulsar-pulsar-client-admin-original-2.9.3.jar:2.9.3]
at org.apache.pulsar.client.admin.internal.BaseResource$1.failed(BaseResource.java:130) ~[org.apache.pulsar-pulsar-client-admin-original-2.9.3.jar:2.9.3]
at org.glassfish.jersey.client.JerseyInvocation$1.failed(JerseyInvocation.java:882) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.glassfish.jersey.client.ClientRuntime.processFailure(ClientRuntime.java:247) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.glassfish.jersey.client.ClientRuntime.processFailure(ClientRuntime.java:242) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.glassfish.jersey.client.ClientRuntime.access$100(ClientRuntime.java:62) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.glassfish.jersey.client.ClientRuntime$2.lambda$failure$1(ClientRuntime.java:178) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) ~[org.glassfish.jersey.core-jersey-common-2.34.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) ~[org.glassfish.jersey.core-jersey-common-2.34.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:292) ~[org.glassfish.jersey.core-jersey-common-2.34.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:274) ~[org.glassfish.jersey.core-jersey-common-2.34.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:244) ~[org.glassfish.jersey.core-jersey-common-2.34.jar:?]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:288) ~[org.glassfish.jersey.core-jersey-common-2.34.jar:?]
at org.glassfish.jersey.client.ClientRuntime$2.failure(ClientRuntime.java:178) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector.lambda$apply$1(AsyncHttpConnector.java:218) ~[org.apache.pulsar-pulsar-client-admin-original-2.9.3.jar:2.9.3]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?]
at org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector.lambda$retryOperation$4(AsyncHttpConnector.java:277) ~[org.apache.pulsar-pulsar-client-admin-original-2.9.3.jar:2.9.3]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?]
at org.asynchttpclient.netty.NettyResponseFuture.abort(NettyResponseFuture.java:273) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender.abort(NettyRequestSender.java:473) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender$1.onFailure(NettyRequestSender.java:325) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.SimpleFutureListener.operationComplete(SimpleFutureListener.java:26) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:552) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:184) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at org.asynchttpclient.netty.request.NettyRequestSender.sendRequestWithNewChannel(NettyRequestSender.java:301) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender.sendRequestWithCertainForceConnect(NettyRequestSender.java:142) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender.sendRequest(NettyRequestSender.java:113) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender.sendNextRequest(NettyRequestSender.java:548) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.handler.intercept.Redirect30xInterceptor.exitAfterHandlingRedirect(Redirect30xInterceptor.java:160) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.handler.intercept.Interceptors.exitAfterIntercept(Interceptors.java:98) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.handler.HttpHandler.handleHttpResponse(HttpHandler.java:78) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.handler.HttpHandler.handleRead(HttpHandler.java:140) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.handler.AsyncHttpClientHandler.channelRead(AsyncHttpClientHandler.java:78) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) ~[io.netty-netty-codec-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327) ~[io.netty-netty-codec-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:299) ~[io.netty-netty-codec-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) ~[io.netty-netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:995) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at java.lang.Thread.run(Thread.java:829) ~[?:?]
Caused by: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:346) ~[?:?]
at java.util.concurrent.CompletableFuture$OrApply.tryFire(CompletableFuture.java:1503) ~[?:?]
... 52 more
Caused by: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
at org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector.lambda$retryOperation$4(AsyncHttpConnector.java:279) ~[org.apache.pulsar-pulsar-client-admin-original-2.9.3.jar:2.9.3]
... 49 more
Caused by: java.net.UnknownHostException: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
at java.net.InetAddress$CachedAddresses.get(InetAddress.java:797) ~[?:?]
at java.net.InetAddress.getAllByName0(InetAddress.java:1509) ~[?:?]
at java.net.InetAddress.getAllByName(InetAddress.java:1368) ~[?:?]
at java.net.InetAddress.getAllByName(InetAddress.java:1302) ~[?:?]
at io.netty.util.internal.SocketUtils$9.run(SocketUtils.java:169) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.internal.SocketUtils$9.run(SocketUtils.java:166) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at java.security.AccessController.doPrivileged(Native Method) ~[?:?]
at io.netty.util.internal.SocketUtils.allAddressesByName(SocketUtils.java:166) ~[io.netty-netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.resolver.DefaultNameResolver.doResolveAll(DefaultNameResolver.java:50) ~[io.netty-netty-resolver-4.1.77.Final.jar:4.1.77.Final]
at io.netty.resolver.SimpleNameResolver.resolveAll(SimpleNameResolver.java:79) ~[io.netty-netty-resolver-4.1.77.Final.jar:4.1.77.Final]
at io.netty.resolver.SimpleNameResolver.resolveAll(SimpleNameResolver.java:71) ~[io.netty-netty-resolver-4.1.77.Final.jar:4.1.77.Final]
at org.asynchttpclient.resolver.RequestHostnameResolver.resolve(RequestHostnameResolver.java:50) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender.resolveAddresses(NettyRequestSender.java:357) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender.sendRequestWithNewChannel(NettyRequestSender.java:300) ~[org.asynchttpclient-async-http-client-2.12.1.jar:?]
... 35 more
2022-08-24T06:55:18,647+0000 [main] ERROR org.apache.pulsar.broker.PulsarService - Failed to start Pulsar service: org.apache.pulsar.client.admin.PulsarAdminException: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
java.lang.RuntimeException: org.apache.pulsar.client.admin.PulsarAdminException: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:581) ~[org.apache.pulsar-pulsar-functions-worker-2.9.3.jar:2.9.3]
at org.apache.pulsar.broker.PulsarService.startWorkerService(PulsarService.java:1579) ~[org.apache.pulsar-pulsar-broker-2.9.3.jar:2.9.3]
at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:769) [org.apache.pulsar-pulsar-broker-2.9.3.jar:2.9.3]
at org.apache.pulsar.PulsarBrokerStarter$BrokerStarter.start(PulsarBrokerStarter.java:273) [org.apache.pulsar-pulsar-broker-2.9.3.jar:2.9.3]
at org.apache.pulsar.PulsarBrokerStarter.main(PulsarBrokerStarter.java:350) [org.apache.pulsar-pulsar-broker-2.9.3.jar:2.9.3]
Caused by: org.apache.pulsar.client.admin.PulsarAdminException: java.util.concurrent.CompletionException: org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$RetryException: Could not complete the operation. Number of retries has been exhausted. Failed reason: pulsar-broker-1.pulsar-broker.pulsar.svc.cluster.local
at org.apache.pulsar.client.admin.internal.BaseResource.getApiException(BaseResource.java:247) ~[org.apache.pulsar-pulsar-client-admin-original-2.9.3.jar:2.9.3]
at org.apache.pulsar.client.admin.internal.BaseResource$1.failed(BaseResource.java:130) ~[org.apache.pulsar-pulsar-client-admin-original-2.9.3.jar:2.9.3]
at org.glassfish.jersey.client.JerseyInvocation$1.failed(JerseyInvocation.java:882) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.glassfish.jersey.client.ClientRuntime.processFailure(ClientRuntime.java:247) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
at org.glassfish.jersey.client.ClientRuntime.processFailure(ClientRuntime.java:242) ~[org.glassfish.jersey.core-jersey-client-2.34.jar:?]
```
```
# kubectl get pod -n pulsar
NAME READY STATUS RESTARTS AGE
pulsar-bookie-0 1/1 Running 0 8d
pulsar-bookie-1 1/1 Running 0 8d
pulsar-bookie-2 1/1 Running 0 8d
pulsar-bookie-3 1/1 Running 0 8d
pulsar-broker-0 1/1 Running 3 (8d ago) 8d
pulsar-broker-1 1/1 Running 2 (8d ago) 8d
pulsar-broker-2 1/1 Running 3 (8d ago) 8d
pulsar-proxy-0 1/1 Running 0 8d
pulsar-proxy-1 1/1 Running 0 8d
pulsar-proxy-2 1/1 Running 0 8d
pulsar-pulsar-manager-79748c4655-c5rxp 1/1 Running 0 8d
pulsar-recovery-0 1/1 Running 0 8d
pulsar-toolset-0 1/1 Running 0 8d
pulsar-zookeeper-0 1/1 Running 0 8d
pulsar-zookeeper-1 1/1 Running 0 8d
pulsar-zookeeper-2 1/1 Running 0 8d
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@pulsar.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org