You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@geode.apache.org by "Rebecca P (JIRA)" <ji...@apache.org> on 2017/12/20 15:26:02 UTC
[jira] [Updated] (GEODE-4131) Deploy JAR through gfsh causes
OutOfMemoryError rendering cluster unstable
[ https://issues.apache.org/jira/browse/GEODE-4131?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Rebecca P updated GEODE-4131:
-----------------------------
Description:
Given a locator with a 512MB max heap using GemFire 9.2.0, we have broken our cluster by uploading a 39MB jar using `gfsh deploy --jar`. We receive the following error:
{{Cluster-0 gfsh>deploy --jar /tmp/plus20.jar
The HTTP request failed with: 500 - org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:978)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.commons.io.output.ByteArrayOutputStream.toByteArray(ByteArrayOutputStream.java:366)
at org.apache.commons.io.IOUtils.toByteArray(IOUtils.java:722)
at org.apache.commons.io.FileUtils.readFileToByteArray(FileUtils.java:1816)
at org.apache.geode.distributed.internal.ClusterConfigurationService.getJarBytesFromThisLocator(ClusterConfigurationService.java:381)
at org.apache.geode.management.internal.configuration.functions.UploadJarFunction.execute(UploadJarFunction.java:43)
at org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:186)
at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
at org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:440)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:666)
at org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1112)
... 1 more
.
java.lang.RuntimeException: The HTTP request failed with: 500 - org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:978)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.commons.io.output.ByteArrayOutputStream.toByteArray(ByteArrayOutputStream.java:366)
at org.apache.commons.io.IOUtils.toByteArray(IOUtils.java:722)
at org.apache.commons.io.FileUtils.readFileToByteArray(FileUtils.java:1816)
at org.apache.geode.distributed.internal.ClusterConfigurationService.getJarBytesFromThisLocator(ClusterConfigurationService.java:381)
at org.apache.geode.management.internal.configuration.functions.UploadJarFunction.execute(UploadJarFunction.java:43)
at org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:186)
at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
at org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:440)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:666)
at org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1112)
... 1 more
.
at org.apache.geode.management.internal.web.http.support.HttpRequester$1.handleError(HttpRequester.java:112)
at org.springframework.web.client.RestTemplate.handleResponse(RestTemplate.java:700)
at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:653)
at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:628)
at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:549)
at org.apache.geode.management.internal.web.http.support.HttpRequester.exchange(HttpRequester.java:144)
at org.apache.geode.management.internal.web.http.support.HttpRequester.post(HttpRequester.java:134)
at org.apache.geode.management.internal.web.shell.HttpOperationInvoker.processCommand(HttpOperationInvoker.java:427)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.executeOnRemote(GfshExecutionStrategy.java:185)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.execute(GfshExecutionStrategy.java:88)
at org.springframework.shell.core.AbstractShell.executeCommand(AbstractShell.java:134)
at org.apache.geode.management.internal.cli.shell.Gfsh.promptLoop(Gfsh.java:1001)
at org.springframework.shell.core.JLineShell.run(JLineShell.java:179)
at java.lang.Thread.run(Thread.java:748)
Could not read command response. Error occurred while executing "deploy --jar /tmp/plus20.jar" on manager.
}}
Once this occurs we are able to connect to the cluster however we continue to receive errors when doing operations like `gfsh list members`
{{
Cluster-0 gfsh>list members
The HTTP request failed with: 500 - org.apache.shiro.UnavailableSecurityManagerException: No SecurityManager accessible to the calling code, either bound to the org.apache.shiro.util.ThreadContext or as a vm static singleton. This is an invalid application configuration.
at org.apache.shiro.SecurityUtils.getSecurityManager(SecurityUtils.java:123)
at org.apache.shiro.subject.Subject$Builder.<init>(Subject.java:627)
at org.apache.shiro.SecurityUtils.getSubject(SecurityUtils.java:56)
at org.apache.geode.internal.security.IntegratedSecurityService.login(IntegratedSecurityService.java:135)
at org.apache.geode.management.internal.web.controllers.support.LoginHandlerInterceptor.preHandle(LoginHandlerInterceptor.java:103)
at org.springframework.web.servlet.HandlerExecutionChain.applyPreHandle(HandlerExecutionChain.java:134)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:958)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
.
java.lang.RuntimeException: The HTTP request failed with: 500 - org.apache.shiro.UnavailableSecurityManagerException: No SecurityManager accessible to the calling code, either bound to the org.apache.shiro.util.ThreadContext or as a vm static singleton. This is an invalid application configuration.
at org.apache.shiro.SecurityUtils.getSecurityManager(SecurityUtils.java:123)
at org.apache.shiro.subject.Subject$Builder.<init>(Subject.java:627)
at org.apache.shiro.SecurityUtils.getSubject(SecurityUtils.java:56)
at org.apache.geode.internal.security.IntegratedSecurityService.login(IntegratedSecurityService.java:135)
at org.apache.geode.management.internal.web.controllers.support.LoginHandlerInterceptor.preHandle(LoginHandlerInterceptor.java:103)
at org.springframework.web.servlet.HandlerExecutionChain.applyPreHandle(HandlerExecutionChain.java:134)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:958)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
.
at org.apache.geode.management.internal.web.http.support.HttpRequester$1.handleError(HttpRequester.java:112)
at org.springframework.web.client.RestTemplate.handleResponse(RestTemplate.java:700)
at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:653)
at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:628)
at org.apache.geode.management.internal.web.http.support.HttpRequester.executeWithResponseExtractor(HttpRequester.java:149)
at org.apache.geode.management.internal.web.shell.HttpOperationInvoker.processCommand(HttpOperationInvoker.java:431)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.executeOnRemote(GfshExecutionStrategy.java:185)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.execute(GfshExecutionStrategy.java:88)
at org.springframework.shell.core.AbstractShell.executeCommand(AbstractShell.java:134)
at org.apache.geode.management.internal.cli.shell.Gfsh.promptLoop(Gfsh.java:1001)
at org.springframework.shell.core.JLineShell.run(JLineShell.java:179)
at java.lang.Thread.run(Thread.java:748)
Could not read command response. Error occurred while executing "list members" on manager.}}
Our locator log has the following error:
{{
[error 2017/12/19 22:22:10.257 UTC locator-b38fd345-4119-403f-85e4-08062352a0b8 <P2P message reader for 10.0.8.28(locator-1344a2b0-0ddd-40d6-a4bb-edf1de102c6b:1:locator)<ec><v39>:49152 unshared ordered uid=83 dom #1 port=36898> tid=0x126] Unable to add jar: plus20.jar
java.lang.ClassCastException: org.apache.geode.cache.execute.FunctionInvocationTargetException cannot be cast to [B
at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174)
at java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1351)
at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)
at org.apache.geode.distributed.internal.ClusterConfigurationService.downloadJarFromLocator(ClusterConfigurationService.java:680)
at org.apache.geode.distributed.internal.ClusterConfigurationService.lambda$downloadJarFromOtherLocators$1(ClusterConfigurationService.java:397)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at java.util.HashMap$KeySpliterator.tryAdvance(HashMap.java:1569)
at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)
at org.apache.geode.distributed.internal.ClusterConfigurationService.downloadJarFromOtherLocators(ClusterConfigurationService.java:398)
at org.apache.geode.management.internal.configuration.callbacks.ConfigurationChangeListener.addOrRemoveJarFromFilesystem(ConfigurationChangeListener.java:79)
at org.apache.geode.management.internal.configuration.callbacks.ConfigurationChangeListener.afterUpdate(ConfigurationChangeListener.java:47)
at org.apache.geode.internal.cache.EnumListenerEvent$AFTER_UPDATE.dispatchEvent(EnumListenerEvent.java:119)
at org.apache.geode.internal.cache.LocalRegion.dispatchEvent(LocalRegion.java:8397)
at org.apache.geode.internal.cache.LocalRegion.dispatchListenerEvent(LocalRegion.java:6911)
at org.apache.geode.internal.cache.LocalRegion.invokePutCallbacks(LocalRegion.java:5901)
at org.apache.geode.internal.cache.EntryEventImpl.invokeCallbacks(EntryEventImpl.java:2298)
at org.apache.geode.internal.cache.AbstractRegionEntry.dispatchListenerEvents(AbstractRegionEntry.java:153)
at org.apache.geode.internal.cache.LocalRegion.basicPutPart2(LocalRegion.java:5760)
at org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2835)
at org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5598)
at org.apache.geode.internal.cache.DistributedRegion.virtualPut(DistributedRegion.java:370)
at org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:151)
at org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5579)
at org.apache.geode.internal.cache.AbstractUpdateOperation.doPutOrCreate(AbstractUpdateOperation.java:165)
at org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.basicOperateOnRegion(AbstractUpdateOperation.java:272)
at org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.operateOnRegion(AbstractUpdateOperation.java:243)
at org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.basicProcess(DistributedCacheOperation.java:1190)
at org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.process(DistributedCacheOperation.java:1091)
at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
at org.apache.geode.distributed.internal.DistributionMessage.schedule(DistributionMessage.java:432)
at org.apache.geode.distributed.internal.DistributionManager.scheduleIncomingMessage(DistributionManager.java:3552)
at org.apache.geode.distributed.internal.DistributionManager.handleIncomingDMsg(DistributionManager.java:3186)
at org.apache.geode.distributed.internal.DistributionManager$MyListener.messageReceived(DistributionManager.java:4361)
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.dispatchMessage(GMSMembershipManager.java:1127)
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.handleOrDeferMessage(GMSMembershipManager.java:1045)
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager$MyDCReceiver.messageReceived(GMSMembershipManager.java:408)
at org.apache.geode.distributed.internal.direct.DirectChannel.receive(DirectChannel.java:714)
at org.apache.geode.internal.tcp.TCPConduit.messageReceived(TCPConduit.java:874)
at org.apache.geode.internal.tcp.Connection.dispatchMessage(Connection.java:3966)
at org.apache.geode.internal.tcp.Connection.processNIOBuffer(Connection.java:3552)
at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1828)
at org.apache.geode.internal.tcp.Connection.run(Connection.java:1689)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
}}
This the app we used to make the 39 MB jar
{{
package eventListener;
import org.apache.geode.cache.Operation;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.asyncqueue.AsyncEvent;
import org.apache.geode.cache.asyncqueue.AsyncEventListener;
import org.apache.geode.internal.logging.LogService;
import org.apache.logging.log4j.Logger;
import java.io.FileInputStream;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.Date;
import java.util.List;
import java.util.Properties;
public class WriteBehindEventListener implements AsyncEventListener {
@Override
public boolean processEvents(List<AsyncEvent> events) {
try {
final Properties prop = this.getConfigProperties();
final String url = "jdbc:mysql://" + prop.getProperty("mySQLDBHostname") + ":" + prop.getProperty("mySQLDBPort") + "/" + prop.getProperty("mySQLDBDatabaseName");
final String user = prop.getProperty("mySQLDBUser");
final String password = prop.getProperty("mySQLDBPassword");
final String tableName = prop.getProperty("mySQLDBTableName");
final String query = "insert into " + tableName + " (somekey, somevalue) values (?, ?)";
for (AsyncEvent event: events) {
if (event.getOperation() == Operation.CREATE) {
final Connection con = DriverManager.getConnection(url, user, password);
final Object key = event.getKey();
final Object value = event.getDeserializedValue();
final PreparedStatement preparedStmt = con.prepareStatement(query);
preparedStmt.setString (1, key.toString());
preparedStmt.setString (2, value.toString());
preparedStmt.execute();
con.close();
}
}
} catch (Exception e) {
System.out.println("PCC-STRING " + (new Date()) + ": " + e.getMessage());
}
return true;
}
@Override
public void close() {
}
private Properties getConfigProperties() throws Exception {
Properties prop = new Properties();
InputStream input = this.getClass().getClassLoader().getResourceAsStream("config.properties");
prop.load(input);
return prop;
}
}
}}
was:
Given a locator with a 512MB max heap using GemFire 9.2.0, we have broken our cluster by uploading a 39MB jar using `gfsh deploy --jar`. We receive the following error:
{{
Cluster-0 gfsh>deploy --jar /tmp/plus20.jar
The HTTP request failed with: 500 - org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:978)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.commons.io.output.ByteArrayOutputStream.toByteArray(ByteArrayOutputStream.java:366)
at org.apache.commons.io.IOUtils.toByteArray(IOUtils.java:722)
at org.apache.commons.io.FileUtils.readFileToByteArray(FileUtils.java:1816)
at org.apache.geode.distributed.internal.ClusterConfigurationService.getJarBytesFromThisLocator(ClusterConfigurationService.java:381)
at org.apache.geode.management.internal.configuration.functions.UploadJarFunction.execute(UploadJarFunction.java:43)
at org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:186)
at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
at org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:440)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:666)
at org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1112)
... 1 more
.
java.lang.RuntimeException: The HTTP request failed with: 500 - org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:978)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.commons.io.output.ByteArrayOutputStream.toByteArray(ByteArrayOutputStream.java:366)
at org.apache.commons.io.IOUtils.toByteArray(IOUtils.java:722)
at org.apache.commons.io.FileUtils.readFileToByteArray(FileUtils.java:1816)
at org.apache.geode.distributed.internal.ClusterConfigurationService.getJarBytesFromThisLocator(ClusterConfigurationService.java:381)
at org.apache.geode.management.internal.configuration.functions.UploadJarFunction.execute(UploadJarFunction.java:43)
at org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:186)
at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
at org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:440)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:666)
at org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1112)
... 1 more
.
at org.apache.geode.management.internal.web.http.support.HttpRequester$1.handleError(HttpRequester.java:112)
at org.springframework.web.client.RestTemplate.handleResponse(RestTemplate.java:700)
at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:653)
at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:628)
at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:549)
at org.apache.geode.management.internal.web.http.support.HttpRequester.exchange(HttpRequester.java:144)
at org.apache.geode.management.internal.web.http.support.HttpRequester.post(HttpRequester.java:134)
at org.apache.geode.management.internal.web.shell.HttpOperationInvoker.processCommand(HttpOperationInvoker.java:427)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.executeOnRemote(GfshExecutionStrategy.java:185)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.execute(GfshExecutionStrategy.java:88)
at org.springframework.shell.core.AbstractShell.executeCommand(AbstractShell.java:134)
at org.apache.geode.management.internal.cli.shell.Gfsh.promptLoop(Gfsh.java:1001)
at org.springframework.shell.core.JLineShell.run(JLineShell.java:179)
at java.lang.Thread.run(Thread.java:748)
Could not read command response. Error occurred while executing "deploy --jar /tmp/plus20.jar" on manager.
}}
Once this occurs we are able to connect to the cluster however we continue to receive errors when doing operations like `gfsh list members`
{{
Cluster-0 gfsh>list members
The HTTP request failed with: 500 - org.apache.shiro.UnavailableSecurityManagerException: No SecurityManager accessible to the calling code, either bound to the org.apache.shiro.util.ThreadContext or as a vm static singleton. This is an invalid application configuration.
at org.apache.shiro.SecurityUtils.getSecurityManager(SecurityUtils.java:123)
at org.apache.shiro.subject.Subject$Builder.<init>(Subject.java:627)
at org.apache.shiro.SecurityUtils.getSubject(SecurityUtils.java:56)
at org.apache.geode.internal.security.IntegratedSecurityService.login(IntegratedSecurityService.java:135)
at org.apache.geode.management.internal.web.controllers.support.LoginHandlerInterceptor.preHandle(LoginHandlerInterceptor.java:103)
at org.springframework.web.servlet.HandlerExecutionChain.applyPreHandle(HandlerExecutionChain.java:134)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:958)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
.
java.lang.RuntimeException: The HTTP request failed with: 500 - org.apache.shiro.UnavailableSecurityManagerException: No SecurityManager accessible to the calling code, either bound to the org.apache.shiro.util.ThreadContext or as a vm static singleton. This is an invalid application configuration.
at org.apache.shiro.SecurityUtils.getSecurityManager(SecurityUtils.java:123)
at org.apache.shiro.subject.Subject$Builder.<init>(Subject.java:627)
at org.apache.shiro.SecurityUtils.getSubject(SecurityUtils.java:56)
at org.apache.geode.internal.security.IntegratedSecurityService.login(IntegratedSecurityService.java:135)
at org.apache.geode.management.internal.web.controllers.support.LoginHandlerInterceptor.preHandle(LoginHandlerInterceptor.java:103)
at org.springframework.web.servlet.HandlerExecutionChain.applyPreHandle(HandlerExecutionChain.java:134)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:958)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:524)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:745)
.
at org.apache.geode.management.internal.web.http.support.HttpRequester$1.handleError(HttpRequester.java:112)
at org.springframework.web.client.RestTemplate.handleResponse(RestTemplate.java:700)
at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:653)
at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:628)
at org.apache.geode.management.internal.web.http.support.HttpRequester.executeWithResponseExtractor(HttpRequester.java:149)
at org.apache.geode.management.internal.web.shell.HttpOperationInvoker.processCommand(HttpOperationInvoker.java:431)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.executeOnRemote(GfshExecutionStrategy.java:185)
at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.execute(GfshExecutionStrategy.java:88)
at org.springframework.shell.core.AbstractShell.executeCommand(AbstractShell.java:134)
at org.apache.geode.management.internal.cli.shell.Gfsh.promptLoop(Gfsh.java:1001)
at org.springframework.shell.core.JLineShell.run(JLineShell.java:179)
at java.lang.Thread.run(Thread.java:748)
Could not read command response. Error occurred while executing "list members" on manager.
}}
Our locator log has the following error:
{{
[error 2017/12/19 22:22:10.257 UTC locator-b38fd345-4119-403f-85e4-08062352a0b8 <P2P message reader for 10.0.8.28(locator-1344a2b0-0ddd-40d6-a4bb-edf1de102c6b:1:locator)<ec><v39>:49152 unshared ordered uid=83 dom #1 port=36898> tid=0x126] Unable to add jar: plus20.jar
java.lang.ClassCastException: org.apache.geode.cache.execute.FunctionInvocationTargetException cannot be cast to [B
at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174)
at java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1351)
at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)
at org.apache.geode.distributed.internal.ClusterConfigurationService.downloadJarFromLocator(ClusterConfigurationService.java:680)
at org.apache.geode.distributed.internal.ClusterConfigurationService.lambda$downloadJarFromOtherLocators$1(ClusterConfigurationService.java:397)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at java.util.HashMap$KeySpliterator.tryAdvance(HashMap.java:1569)
at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)
at org.apache.geode.distributed.internal.ClusterConfigurationService.downloadJarFromOtherLocators(ClusterConfigurationService.java:398)
at org.apache.geode.management.internal.configuration.callbacks.ConfigurationChangeListener.addOrRemoveJarFromFilesystem(ConfigurationChangeListener.java:79)
at org.apache.geode.management.internal.configuration.callbacks.ConfigurationChangeListener.afterUpdate(ConfigurationChangeListener.java:47)
at org.apache.geode.internal.cache.EnumListenerEvent$AFTER_UPDATE.dispatchEvent(EnumListenerEvent.java:119)
at org.apache.geode.internal.cache.LocalRegion.dispatchEvent(LocalRegion.java:8397)
at org.apache.geode.internal.cache.LocalRegion.dispatchListenerEvent(LocalRegion.java:6911)
at org.apache.geode.internal.cache.LocalRegion.invokePutCallbacks(LocalRegion.java:5901)
at org.apache.geode.internal.cache.EntryEventImpl.invokeCallbacks(EntryEventImpl.java:2298)
at org.apache.geode.internal.cache.AbstractRegionEntry.dispatchListenerEvents(AbstractRegionEntry.java:153)
at org.apache.geode.internal.cache.LocalRegion.basicPutPart2(LocalRegion.java:5760)
at org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2835)
at org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5598)
at org.apache.geode.internal.cache.DistributedRegion.virtualPut(DistributedRegion.java:370)
at org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:151)
at org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5579)
at org.apache.geode.internal.cache.AbstractUpdateOperation.doPutOrCreate(AbstractUpdateOperation.java:165)
at org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.basicOperateOnRegion(AbstractUpdateOperation.java:272)
at org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.operateOnRegion(AbstractUpdateOperation.java:243)
at org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.basicProcess(DistributedCacheOperation.java:1190)
at org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.process(DistributedCacheOperation.java:1091)
at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
at org.apache.geode.distributed.internal.DistributionMessage.schedule(DistributionMessage.java:432)
at org.apache.geode.distributed.internal.DistributionManager.scheduleIncomingMessage(DistributionManager.java:3552)
at org.apache.geode.distributed.internal.DistributionManager.handleIncomingDMsg(DistributionManager.java:3186)
at org.apache.geode.distributed.internal.DistributionManager$MyListener.messageReceived(DistributionManager.java:4361)
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.dispatchMessage(GMSMembershipManager.java:1127)
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.handleOrDeferMessage(GMSMembershipManager.java:1045)
at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager$MyDCReceiver.messageReceived(GMSMembershipManager.java:408)
at org.apache.geode.distributed.internal.direct.DirectChannel.receive(DirectChannel.java:714)
at org.apache.geode.internal.tcp.TCPConduit.messageReceived(TCPConduit.java:874)
at org.apache.geode.internal.tcp.Connection.dispatchMessage(Connection.java:3966)
at org.apache.geode.internal.tcp.Connection.processNIOBuffer(Connection.java:3552)
at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1828)
at org.apache.geode.internal.tcp.Connection.run(Connection.java:1689)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
}}
This the app we used to make the 39 MB jar
{{
package eventListener;
import org.apache.geode.cache.Operation;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.asyncqueue.AsyncEvent;
import org.apache.geode.cache.asyncqueue.AsyncEventListener;
import org.apache.geode.internal.logging.LogService;
import org.apache.logging.log4j.Logger;
import java.io.FileInputStream;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.util.Date;
import java.util.List;
import java.util.Properties;
public class WriteBehindEventListener implements AsyncEventListener {
@Override
public boolean processEvents(List<AsyncEvent> events) {
try {
final Properties prop = this.getConfigProperties();
final String url = "jdbc:mysql://" + prop.getProperty("mySQLDBHostname") + ":" + prop.getProperty("mySQLDBPort") + "/" + prop.getProperty("mySQLDBDatabaseName");
final String user = prop.getProperty("mySQLDBUser");
final String password = prop.getProperty("mySQLDBPassword");
final String tableName = prop.getProperty("mySQLDBTableName");
final String query = "insert into " + tableName + " (somekey, somevalue) values (?, ?)";
for (AsyncEvent event: events) {
if (event.getOperation() == Operation.CREATE) {
final Connection con = DriverManager.getConnection(url, user, password);
final Object key = event.getKey();
final Object value = event.getDeserializedValue();
final PreparedStatement preparedStmt = con.prepareStatement(query);
preparedStmt.setString (1, key.toString());
preparedStmt.setString (2, value.toString());
preparedStmt.execute();
con.close();
}
}
} catch (Exception e) {
System.out.println("PCC-STRING " + (new Date()) + ": " + e.getMessage());
}
return true;
}
@Override
public void close() {
}
private Properties getConfigProperties() throws Exception {
Properties prop = new Properties();
InputStream input = this.getClass().getClassLoader().getResourceAsStream("config.properties");
prop.load(input);
return prop;
}
}
}}
> Deploy JAR through gfsh causes OutOfMemoryError rendering cluster unstable
> --------------------------------------------------------------------------
>
> Key: GEODE-4131
> URL: https://issues.apache.org/jira/browse/GEODE-4131
> Project: Geode
> Issue Type: Bug
> Components: gfsh, locator
> Reporter: Rebecca P
>
> Given a locator with a 512MB max heap using GemFire 9.2.0, we have broken our cluster by uploading a 39MB jar using `gfsh deploy --jar`. We receive the following error:
> {{Cluster-0 gfsh>deploy --jar /tmp/plus20.jar
> The HTTP request failed with: 500 - org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space
> at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:978)
> at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
> at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
> at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
> at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
> at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
> at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
> at org.eclipse.jetty.server.Server.handle(Server.java:524)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.OutOfMemoryError: Java heap space
> at org.apache.commons.io.output.ByteArrayOutputStream.toByteArray(ByteArrayOutputStream.java:366)
> at org.apache.commons.io.IOUtils.toByteArray(IOUtils.java:722)
> at org.apache.commons.io.FileUtils.readFileToByteArray(FileUtils.java:1816)
> at org.apache.geode.distributed.internal.ClusterConfigurationService.getJarBytesFromThisLocator(ClusterConfigurationService.java:381)
> at org.apache.geode.management.internal.configuration.functions.UploadJarFunction.execute(UploadJarFunction.java:43)
> at org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:186)
> at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
> at org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:440)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:666)
> at org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1112)
> ... 1 more
> .
> java.lang.RuntimeException: The HTTP request failed with: 500 - org.springframework.web.util.NestedServletException: Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space
> at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:978)
> at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
> at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
> at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
> at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
> at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
> at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
> at org.eclipse.jetty.server.Server.handle(Server.java:524)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.OutOfMemoryError: Java heap space
> at org.apache.commons.io.output.ByteArrayOutputStream.toByteArray(ByteArrayOutputStream.java:366)
> at org.apache.commons.io.IOUtils.toByteArray(IOUtils.java:722)
> at org.apache.commons.io.FileUtils.readFileToByteArray(FileUtils.java:1816)
> at org.apache.geode.distributed.internal.ClusterConfigurationService.getJarBytesFromThisLocator(ClusterConfigurationService.java:381)
> at org.apache.geode.management.internal.configuration.functions.UploadJarFunction.execute(UploadJarFunction.java:43)
> at org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:186)
> at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
> at org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:440)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:666)
> at org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1112)
> ... 1 more
> .
> at org.apache.geode.management.internal.web.http.support.HttpRequester$1.handleError(HttpRequester.java:112)
> at org.springframework.web.client.RestTemplate.handleResponse(RestTemplate.java:700)
> at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:653)
> at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:628)
> at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:549)
> at org.apache.geode.management.internal.web.http.support.HttpRequester.exchange(HttpRequester.java:144)
> at org.apache.geode.management.internal.web.http.support.HttpRequester.post(HttpRequester.java:134)
> at org.apache.geode.management.internal.web.shell.HttpOperationInvoker.processCommand(HttpOperationInvoker.java:427)
> at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.executeOnRemote(GfshExecutionStrategy.java:185)
> at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.execute(GfshExecutionStrategy.java:88)
> at org.springframework.shell.core.AbstractShell.executeCommand(AbstractShell.java:134)
> at org.apache.geode.management.internal.cli.shell.Gfsh.promptLoop(Gfsh.java:1001)
> at org.springframework.shell.core.JLineShell.run(JLineShell.java:179)
> at java.lang.Thread.run(Thread.java:748)
> Could not read command response. Error occurred while executing "deploy --jar /tmp/plus20.jar" on manager.
> }}
> Once this occurs we are able to connect to the cluster however we continue to receive errors when doing operations like `gfsh list members`
> {{
> Cluster-0 gfsh>list members
> The HTTP request failed with: 500 - org.apache.shiro.UnavailableSecurityManagerException: No SecurityManager accessible to the calling code, either bound to the org.apache.shiro.util.ThreadContext or as a vm static singleton. This is an invalid application configuration.
> at org.apache.shiro.SecurityUtils.getSecurityManager(SecurityUtils.java:123)
> at org.apache.shiro.subject.Subject$Builder.<init>(Subject.java:627)
> at org.apache.shiro.SecurityUtils.getSubject(SecurityUtils.java:56)
> at org.apache.geode.internal.security.IntegratedSecurityService.login(IntegratedSecurityService.java:135)
> at org.apache.geode.management.internal.web.controllers.support.LoginHandlerInterceptor.preHandle(LoginHandlerInterceptor.java:103)
> at org.springframework.web.servlet.HandlerExecutionChain.applyPreHandle(HandlerExecutionChain.java:134)
> at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:958)
> at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
> at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
> at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
> at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
> at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
> at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
> at org.eclipse.jetty.server.Server.handle(Server.java:524)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
> at java.lang.Thread.run(Thread.java:745)
> .
> java.lang.RuntimeException: The HTTP request failed with: 500 - org.apache.shiro.UnavailableSecurityManagerException: No SecurityManager accessible to the calling code, either bound to the org.apache.shiro.util.ThreadContext or as a vm static singleton. This is an invalid application configuration.
> at org.apache.shiro.SecurityUtils.getSecurityManager(SecurityUtils.java:123)
> at org.apache.shiro.subject.Subject$Builder.<init>(Subject.java:627)
> at org.apache.shiro.SecurityUtils.getSubject(SecurityUtils.java:56)
> at org.apache.geode.internal.security.IntegratedSecurityService.login(IntegratedSecurityService.java:135)
> at org.apache.geode.management.internal.web.controllers.support.LoginHandlerInterceptor.preHandle(LoginHandlerInterceptor.java:103)
> at org.springframework.web.servlet.HandlerExecutionChain.applyPreHandle(HandlerExecutionChain.java:134)
> at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:958)
> at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
> at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
> at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
> at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
> at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
> at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
> at org.eclipse.jetty.server.Server.handle(Server.java:524)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:319)
> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
> at java.lang.Thread.run(Thread.java:745)
> .
> at org.apache.geode.management.internal.web.http.support.HttpRequester$1.handleError(HttpRequester.java:112)
> at org.springframework.web.client.RestTemplate.handleResponse(RestTemplate.java:700)
> at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:653)
> at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:628)
> at org.apache.geode.management.internal.web.http.support.HttpRequester.executeWithResponseExtractor(HttpRequester.java:149)
> at org.apache.geode.management.internal.web.shell.HttpOperationInvoker.processCommand(HttpOperationInvoker.java:431)
> at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.executeOnRemote(GfshExecutionStrategy.java:185)
> at org.apache.geode.management.internal.cli.shell.GfshExecutionStrategy.execute(GfshExecutionStrategy.java:88)
> at org.springframework.shell.core.AbstractShell.executeCommand(AbstractShell.java:134)
> at org.apache.geode.management.internal.cli.shell.Gfsh.promptLoop(Gfsh.java:1001)
> at org.springframework.shell.core.JLineShell.run(JLineShell.java:179)
> at java.lang.Thread.run(Thread.java:748)
> Could not read command response. Error occurred while executing "list members" on manager.}}
> Our locator log has the following error:
> {{
> [error 2017/12/19 22:22:10.257 UTC locator-b38fd345-4119-403f-85e4-08062352a0b8 <P2P message reader for 10.0.8.28(locator-1344a2b0-0ddd-40d6-a4bb-edf1de102c6b:1:locator)<ec><v39>:49152 unshared ordered uid=83 dom #1 port=36898> tid=0x126] Unable to add jar: plus20.jar
> java.lang.ClassCastException: org.apache.geode.cache.execute.FunctionInvocationTargetException cannot be cast to [B
> at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174)
> at java.util.ArrayList$ArrayListSpliterator.tryAdvance(ArrayList.java:1351)
> at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
> at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
> at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
> at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)
> at org.apache.geode.distributed.internal.ClusterConfigurationService.downloadJarFromLocator(ClusterConfigurationService.java:680)
> at org.apache.geode.distributed.internal.ClusterConfigurationService.lambda$downloadJarFromOtherLocators$1(ClusterConfigurationService.java:397)
> at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> at java.util.HashMap$KeySpliterator.tryAdvance(HashMap.java:1569)
> at java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)
> at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)
> at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
> at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)
> at org.apache.geode.distributed.internal.ClusterConfigurationService.downloadJarFromOtherLocators(ClusterConfigurationService.java:398)
> at org.apache.geode.management.internal.configuration.callbacks.ConfigurationChangeListener.addOrRemoveJarFromFilesystem(ConfigurationChangeListener.java:79)
> at org.apache.geode.management.internal.configuration.callbacks.ConfigurationChangeListener.afterUpdate(ConfigurationChangeListener.java:47)
> at org.apache.geode.internal.cache.EnumListenerEvent$AFTER_UPDATE.dispatchEvent(EnumListenerEvent.java:119)
> at org.apache.geode.internal.cache.LocalRegion.dispatchEvent(LocalRegion.java:8397)
> at org.apache.geode.internal.cache.LocalRegion.dispatchListenerEvent(LocalRegion.java:6911)
> at org.apache.geode.internal.cache.LocalRegion.invokePutCallbacks(LocalRegion.java:5901)
> at org.apache.geode.internal.cache.EntryEventImpl.invokeCallbacks(EntryEventImpl.java:2298)
> at org.apache.geode.internal.cache.AbstractRegionEntry.dispatchListenerEvents(AbstractRegionEntry.java:153)
> at org.apache.geode.internal.cache.LocalRegion.basicPutPart2(LocalRegion.java:5760)
> at org.apache.geode.internal.cache.AbstractRegionMap.basicPut(AbstractRegionMap.java:2835)
> at org.apache.geode.internal.cache.LocalRegion.virtualPut(LocalRegion.java:5598)
> at org.apache.geode.internal.cache.DistributedRegion.virtualPut(DistributedRegion.java:370)
> at org.apache.geode.internal.cache.LocalRegionDataView.putEntry(LocalRegionDataView.java:151)
> at org.apache.geode.internal.cache.LocalRegion.basicUpdate(LocalRegion.java:5579)
> at org.apache.geode.internal.cache.AbstractUpdateOperation.doPutOrCreate(AbstractUpdateOperation.java:165)
> at org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.basicOperateOnRegion(AbstractUpdateOperation.java:272)
> at org.apache.geode.internal.cache.AbstractUpdateOperation$AbstractUpdateMessage.operateOnRegion(AbstractUpdateOperation.java:243)
> at org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.basicProcess(DistributedCacheOperation.java:1190)
> at org.apache.geode.internal.cache.DistributedCacheOperation$CacheOperationMessage.process(DistributedCacheOperation.java:1091)
> at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:374)
> at org.apache.geode.distributed.internal.DistributionMessage.schedule(DistributionMessage.java:432)
> at org.apache.geode.distributed.internal.DistributionManager.scheduleIncomingMessage(DistributionManager.java:3552)
> at org.apache.geode.distributed.internal.DistributionManager.handleIncomingDMsg(DistributionManager.java:3186)
> at org.apache.geode.distributed.internal.DistributionManager$MyListener.messageReceived(DistributionManager.java:4361)
> at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.dispatchMessage(GMSMembershipManager.java:1127)
> at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager.handleOrDeferMessage(GMSMembershipManager.java:1045)
> at org.apache.geode.distributed.internal.membership.gms.mgr.GMSMembershipManager$MyDCReceiver.messageReceived(GMSMembershipManager.java:408)
> at org.apache.geode.distributed.internal.direct.DirectChannel.receive(DirectChannel.java:714)
> at org.apache.geode.internal.tcp.TCPConduit.messageReceived(TCPConduit.java:874)
> at org.apache.geode.internal.tcp.Connection.dispatchMessage(Connection.java:3966)
> at org.apache.geode.internal.tcp.Connection.processNIOBuffer(Connection.java:3552)
> at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1828)
> at org.apache.geode.internal.tcp.Connection.run(Connection.java:1689)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> }}
> This the app we used to make the 39 MB jar
> {{
> package eventListener;
> import org.apache.geode.cache.Operation;
> import org.apache.geode.cache.Region;
> import org.apache.geode.cache.asyncqueue.AsyncEvent;
> import org.apache.geode.cache.asyncqueue.AsyncEventListener;
> import org.apache.geode.internal.logging.LogService;
> import org.apache.logging.log4j.Logger;
> import java.io.FileInputStream;
> import java.io.InputStream;
> import java.sql.Connection;
> import java.sql.DriverManager;
> import java.sql.PreparedStatement;
> import java.util.Date;
> import java.util.List;
> import java.util.Properties;
> public class WriteBehindEventListener implements AsyncEventListener {
> @Override
> public boolean processEvents(List<AsyncEvent> events) {
> try {
> final Properties prop = this.getConfigProperties();
> final String url = "jdbc:mysql://" + prop.getProperty("mySQLDBHostname") + ":" + prop.getProperty("mySQLDBPort") + "/" + prop.getProperty("mySQLDBDatabaseName");
> final String user = prop.getProperty("mySQLDBUser");
> final String password = prop.getProperty("mySQLDBPassword");
> final String tableName = prop.getProperty("mySQLDBTableName");
> final String query = "insert into " + tableName + " (somekey, somevalue) values (?, ?)";
> for (AsyncEvent event: events) {
> if (event.getOperation() == Operation.CREATE) {
> final Connection con = DriverManager.getConnection(url, user, password);
> final Object key = event.getKey();
> final Object value = event.getDeserializedValue();
> final PreparedStatement preparedStmt = con.prepareStatement(query);
> preparedStmt.setString (1, key.toString());
> preparedStmt.setString (2, value.toString());
> preparedStmt.execute();
> con.close();
> }
> }
> } catch (Exception e) {
> System.out.println("PCC-STRING " + (new Date()) + ": " + e.getMessage());
> }
> return true;
> }
> @Override
> public void close() {
> }
> private Properties getConfigProperties() throws Exception {
> Properties prop = new Properties();
> InputStream input = this.getClass().getClassLoader().getResourceAsStream("config.properties");
> prop.load(input);
> return prop;
> }
> }
> }}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)