You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by Susheel Kumar <su...@gmail.com> on 2018/07/23 19:46:49 UTC
Solr fails even ZK quorum has majority
In usual circumstances when one Zookeeper goes down while others 2 are up,
Solr continues to operate but when one of the ZK machine was not reachable
with ping returning below results, Solr count't starts. See stack trace
below
ping: cannot resolve ditsearch001.es.com: Unknown host
Setup: Solr 6.6.2 and Zookeeper 3.4.10
I had to remove this server name from the ZK_HOST list (solr.in.sh) in
order to get Solr started. Ideally whatever issue is there as far as
majority is there, Solr should get started.
Has any one noticed this issue?
Thnx
2018-07-23 15:30:47.218 INFO (main) [ ] o.e.j.s.Server
jetty-9.3.14.v20161028
2018-07-23 15:30:47.817 INFO (main) [ ] o.a.s.s.SolrDispatchFilter ___
_ Welcome to Apache Solr‚Ñ¢ version 6.6.2
2018-07-23 15:30:47.829 INFO (main) [ ] o.a.s.s.SolrDispatchFilter / __|
___| |_ _ Starting in cloud mode on port 8080
2018-07-23 15:30:47.830 INFO (main) [ ] o.a.s.s.SolrDispatchFilter \__
\/ _ \ | '_| Install dir: /opt/solr
2018-07-23 15:30:47.861 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
|___/\___/_|_| Start time: 2018-07-23T15:30:47.832Z
2018-07-23 15:30:47.863 INFO (main) [ ] o.a.s.s.StartupLoggingUtils
Property solr.log.muteconsole given. Muting ConsoleAppender named CONSOLE
2018-07-23 15:30:47.929 INFO (main) [ ] o.a.s.c.SolrResourceLoader Using
system property solr.solr.home: /app/solr/data
2018-07-23 15:30:48.037 ERROR (main) [ ] o.a.s.s.SolrDispatchFilter Could
not start Solr. Check solr/home property and the logs
2018-07-23 15:30:48.235 ERROR (main) [ ] o.a.s.c.SolrCore
null:org.apache.solr.common.SolrException: Error occurred while loading
solr.xml from zookeeper
at
org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(SolrDispatchFilter.java:270)
at
org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:242)
at
org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:173)
at
org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
at
org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
at
org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:349)
at
org.eclipse.jetty.webapp.WebAppContext.startWebapp(WebAppContext.java:1404)
at
org.eclipse.jetty.webapp.WebAppContext.startContext(WebAppContext.java:1366)
at
org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:778)
at
org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:262)
at
org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:520)
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at
org.eclipse.jetty.deploy.bindings.StandardStarter.processBinding(StandardStarter.java:41)
at
org.eclipse.jetty.deploy.AppLifeCycle.runBindings(AppLifeCycle.java:188)
at
org.eclipse.jetty.deploy.DeploymentManager.requestAppGoal(DeploymentManager.java:499)
at
org.eclipse.jetty.deploy.DeploymentManager.addApp(DeploymentManager.java:147)
at
org.eclipse.jetty.deploy.providers.ScanningAppProvider.fileAdded(ScanningAppProvider.java:180)
at
org.eclipse.jetty.deploy.providers.WebAppProvider.fileAdded(WebAppProvider.java:458)
at
org.eclipse.jetty.deploy.providers.ScanningAppProvider$1.fileAdded(ScanningAppProvider.java:64)
at org.eclipse.jetty.util.Scanner.reportAddition(Scanner.java:610)
at
org.eclipse.jetty.util.Scanner.reportDifferences(Scanner.java:529)
at org.eclipse.jetty.util.Scanner.scan(Scanner.java:392)
at org.eclipse.jetty.util.Scanner.doStart(Scanner.java:313)
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at
org.eclipse.jetty.deploy.providers.ScanningAppProvider.doStart(ScanningAppProvider.java:150)
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at
org.eclipse.jetty.deploy.DeploymentManager.startAppProvider(DeploymentManager.java:561)
at
org.eclipse.jetty.deploy.DeploymentManager.doStart(DeploymentManager.java:236)
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at
org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:131)
at org.eclipse.jetty.server.Server.start(Server.java:422)
at
org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:113)
at
org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61)
at org.eclipse.jetty.server.Server.doStart(Server.java:389)
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at
org.eclipse.jetty.xml.XmlConfiguration$1.run(XmlConfiguration.java:1516)
at java.security.AccessController.doPrivileged(Native Method)
at
org.eclipse.jetty.xml.XmlConfiguration.main(XmlConfiguration.java:1441)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.eclipse.jetty.start.Main.invokeMain(Main.java:214)
at org.eclipse.jetty.start.Main.start(Main.java:457)
at org.eclipse.jetty.start.Main.main(Main.java:75)
Caused by: org.apache.solr.common.SolrException:
java.net.UnknownHostException: ditsearch001.es.com: Name or service not
known
at
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
at
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
at
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
at
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
at
org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(SolrDispatchFilter.java:263)
... 44 more
Caused by: java.net.UnknownHostException: ditsearch001.es.com: Name or
service not known
at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
at
java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
at java.net.InetAddress.getAllByName(InetAddress.java:1192)
at java.net.InetAddress.getAllByName(InetAddress.java:1126)
at
org.apache.zookeeper.client.StaticHostProvider.<init>(StaticHostProvider.java:61)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:445)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
at
org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
at
org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
at
org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
at
org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
... 48 more
Re: Solr fails even ZK quorum has majority
Posted by Susheel Kumar <su...@gmail.com>.
Thank you, Shalin.
Here is the Jira https://issues.apache.org/jira/browse/SOLR-12585
On Mon, Jul 23, 2018 at 11:21 PM, Shalin Shekhar Mangar <
shalinmangar@gmail.com> wrote:
> Can you please open a Jira issue? I don't think we handle DNS problems very
> well during startup. Thanks.
>
> On Tue, Jul 24, 2018 at 2:31 AM Susheel Kumar <su...@gmail.com>
> wrote:
>
> > Something messed up with DNS which resulted into unknown host exception
> for
> > one the machines in our env and caused Solr to throw the above exception
> >
> > Eric, I have the Solr configured using service installation script and
> > the ZK_HOST entry in
> > solr.in.sh="server1:2181,server2:2181,server3:2181/collection"
> > and after removing the server1 from above, was able to start Solr
> otherwise
> > it was throwing above exception.
> >
> > Thnx
> >
> >
> > On Mon, Jul 23, 2018 at 4:20 PM, Erick Erickson <erickerickson@gmail.com
> >
> > wrote:
> >
> > > And how do you start Solr? Do you use the entire 3-node ensemble
> address?
> > >
> > > On Mon, Jul 23, 2018 at 12:55 PM, Michael Braun <n3...@gmail.com>
> > wrote:
> > > > Per the exception, this looks like a network / DNS resolution issue,
> > > > independent of Solr and Zookeeper code:
> > > >
> > > > Caused by: org.apache.solr.common.SolrException:
> > > > java.net.UnknownHostException: ditsearch001.es.com: Name or service
> > not
> > > > known
> > > >
> > > > Is this address actually resolvable at the time?
> > > >
> > > > On Mon, Jul 23, 2018 at 3:46 PM, Susheel Kumar <
> susheel2777@gmail.com>
> > > > wrote:
> > > >
> > > >> In usual circumstances when one Zookeeper goes down while others 2
> are
> > > up,
> > > >> Solr continues to operate but when one of the ZK machine was not
> > > reachable
> > > >> with ping returning below results, Solr count't starts. See stack
> > trace
> > > >> below
> > > >>
> > > >> ping: cannot resolve ditsearch001.es.com: Unknown host
> > > >>
> > > >>
> > > >> Setup: Solr 6.6.2 and Zookeeper 3.4.10
> > > >>
> > > >> I had to remove this server name from the ZK_HOST list (solr.in.sh)
> > in
> > > >> order to get Solr started. Ideally whatever issue is there as far as
> > > >> majority is there, Solr should get started.
> > > >>
> > > >> Has any one noticed this issue?
> > > >>
> > > >> Thnx
> > > >>
> > > >> 2018-07-23 15:30:47.218 INFO (main) [ ] o.e.j.s.Server
> > > >> jetty-9.3.14.v20161028
> > > >>
> > > >> 2018-07-23 15:30:47.817 INFO (main) [ ]
> o.a.s.s.SolrDispatchFilter
> > > ___
> > > >> _ Welcome to Apache Solr‚Ñ¢ version 6.6.2
> > > >>
> > > >> 2018-07-23 15:30:47.829 INFO (main) [ ]
> o.a.s.s.SolrDispatchFilter
> > /
> > > __|
> > > >> ___| |_ _ Starting in cloud mode on port 8080
> > > >>
> > > >> 2018-07-23 15:30:47.830 INFO (main) [ ]
> o.a.s.s.SolrDispatchFilter
> > > \__
> > > >> \/ _ \ | '_| Install dir: /opt/solr
> > > >>
> > > >> 2018-07-23 15:30:47.861 INFO (main) [ ]
> o.a.s.s.SolrDispatchFilter
> > > >> |___/\___/_|_| Start time: 2018-07-23T15:30:47.832Z
> > > >>
> > > >> 2018-07-23 15:30:47.863 INFO (main) [ ]
> o.a.s.s.StartupLoggingUtils
> > > >> Property solr.log.muteconsole given. Muting ConsoleAppender named
> > > CONSOLE
> > > >>
> > > >> 2018-07-23 15:30:47.929 INFO (main) [ ]
> o.a.s.c.SolrResourceLoader
> > > Using
> > > >> system property solr.solr.home: /app/solr/data
> > > >>
> > > >> 2018-07-23 15:30:48.037 ERROR (main) [ ]
> o.a.s.s.SolrDispatchFilter
> > > Could
> > > >> not start Solr. Check solr/home property and the logs
> > > >>
> > > >> 2018-07-23 15:30:48.235 ERROR (main) [ ] o.a.s.c.SolrCore
> > > >> null:org.apache.solr.common.SolrException: Error occurred while
> > loading
> > > >> solr.xml from zookeeper
> > > >>
> > > >> at
> > > >> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> > > >> SolrDispatchFilter.java:270)
> > > >>
> > > >> at
> > > >> org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(
> > > >> SolrDispatchFilter.java:242)
> > > >>
> > > >> at
> > > >> org.apache.solr.servlet.SolrDispatchFilter.init(
> > > >> SolrDispatchFilter.java:173)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.servlet.FilterHolder.initialize(
> > > FilterHolder.java:137)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.servlet.ServletHandler.initialize(
> > > >> ServletHandler.java:873)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.servlet.ServletContextHandler.startContext(
> > > >> ServletContextHandler.java:349)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.webapp.WebAppContext.startWebapp(
> > > >> WebAppContext.java:1404)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.webapp.WebAppContext.startContext(
> > > >> WebAppContext.java:1366)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.server.handler.ContextHandler.
> > > >> doStart(ContextHandler.java:778)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.servlet.ServletContextHandler.doStart(
> > > >> ServletContextHandler.java:262)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.webapp.WebAppContext.doStart(
> WebAppContext.java:520)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > > >> start(AbstractLifeCycle.java:68)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.bindings.StandardStarter.processBinding(
> > > >> StandardStarter.java:41)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.AppLifeCycle.runBindings(
> > > AppLifeCycle.java:188)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.DeploymentManager.requestAppGoal(
> > > >> DeploymentManager.java:499)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.DeploymentManager.addApp(
> > > >> DeploymentManager.java:147)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> > > >> fileAdded(ScanningAppProvider.java:180)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.providers.WebAppProvider.
> > > >> fileAdded(WebAppProvider.java:458)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.providers.ScanningAppProvider$1.fileAdded(
> > > >> ScanningAppProvider.java:64)
> > > >>
> > > >> at org.eclipse.jetty.util.Scanner.reportAddition(
> > > Scanner.java:610)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.Scanner.reportDifferences(Scanner.java:529)
> > > >>
> > > >> at org.eclipse.jetty.util.Scanner.scan(Scanner.java:392)
> > > >>
> > > >> at org.eclipse.jetty.util.Scanner.doStart(Scanner.java:313)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > > >> start(AbstractLifeCycle.java:68)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> > > >> doStart(ScanningAppProvider.java:150)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > > >> start(AbstractLifeCycle.java:68)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.DeploymentManager.startAppProvider(
> > > >> DeploymentManager.java:561)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.deploy.DeploymentManager.doStart(
> > > >> DeploymentManager.java:236)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > > >> start(AbstractLifeCycle.java:68)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.component.ContainerLifeCycle.
> > > >> start(ContainerLifeCycle.java:131)
> > > >>
> > > >> at org.eclipse.jetty.server.Server.start(Server.java:422)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.component.ContainerLifeCycle.
> > > >> doStart(ContainerLifeCycle.java:113)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.server.handler.AbstractHandler.
> > > >> doStart(AbstractHandler.java:61)
> > > >>
> > > >> at org.eclipse.jetty.server.Server.doStart(Server.java:389)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > > >> start(AbstractLifeCycle.java:68)
> > > >>
> > > >> at
> > > >> org.eclipse.jetty.xml.XmlConfiguration$1.run(
> > > XmlConfiguration.java:1516)
> > > >>
> > > >> at java.security.AccessController.doPrivileged(Native
> Method)
> > > >>
> > > >> at
> > > >>
> > org.eclipse.jetty.xml.XmlConfiguration.main(XmlConfiguration.java:1441)
> > > >>
> > > >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > > >>
> > > >> at
> > > >> sun.reflect.NativeMethodAccessorImpl.invoke(
> > > NativeMethodAccessorImpl.java:
> > > >> 62)
> > > >>
> > > >> at
> > > >> sun.reflect.DelegatingMethodAccessorImpl.invoke(
> > > >> DelegatingMethodAccessorImpl.java:43)
> > > >>
> > > >> at java.lang.reflect.Method.invoke(Method.java:498)
> > > >>
> > > >> at org.eclipse.jetty.start.Main.invokeMain(Main.java:214)
> > > >>
> > > >> at org.eclipse.jetty.start.Main.start(Main.java:457)
> > > >>
> > > >> at org.eclipse.jetty.start.Main.main(Main.java:75)
> > > >>
> > > >> Caused by: org.apache.solr.common.SolrException:
> > > >> java.net.UnknownHostException: ditsearch001.es.com: Name or service
> > not
> > > >> known
> > > >>
> > > >> at
> > > >>
> > org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
> > > >>
> > > >> at
> > > >>
> > org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
> > > >>
> > > >> at
> > > >>
> > org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
> > > >>
> > > >> at
> > > >> org.apache.solr.common.cloud.SolrZkClient.<init>(
> SolrZkClient.java:99)
> > > >>
> > > >> at
> > > >> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> > > >> SolrDispatchFilter.java:263)
> > > >>
> > > >> ... 44 more
> > > >>
> > > >> Caused by: java.net.UnknownHostException: ditsearch001.es.com: Name
> > or
> > > >> service not known
> > > >>
> > > >> at java.net.Inet6AddressImpl.lookupAllHostAddr(Native
> Method)
> > > >>
> > > >> at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.
> > > java:928)
> > > >>
> > > >> at
> > > >>
> > java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
> > > >>
> > > >> at java.net.InetAddress.getAllByName0(InetAddress.
> java:1276)
> > > >>
> > > >> at java.net.InetAddress.getAllByName(InetAddress.java:1192)
> > > >>
> > > >> at java.net.InetAddress.getAllByName(InetAddress.java:1126)
> > > >>
> > > >> at
> > > >> org.apache.zookeeper.client.StaticHostProvider.<init>(
> > > >> StaticHostProvider.java:61)
> > > >>
> > > >> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.
> java:445)
> > > >>
> > > >> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.
> java:380)
> > > >>
> > > >> at
> > > >> org.apache.solr.common.cloud.SolrZooKeeper.<init>(
> > > SolrZooKeeper.java:43)
> > > >>
> > > >> at
> > > >> org.apache.solr.common.cloud.ZkClientConnectionStrategy.
> > > >> createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
> > > >>
> > > >> at
> > > >> org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(
> > > >> DefaultConnectionStrategy.java:37)
> > > >>
> > > >> at
> > > >>
> > org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
> > > >>
> > > >> ... 48 more
> > > >>
> > >
> >
>
>
> --
> Regards,
> Shalin Shekhar Mangar.
>
Re: Solr fails even ZK quorum has majority
Posted by Shalin Shekhar Mangar <sh...@gmail.com>.
Can you please open a Jira issue? I don't think we handle DNS problems very
well during startup. Thanks.
On Tue, Jul 24, 2018 at 2:31 AM Susheel Kumar <su...@gmail.com> wrote:
> Something messed up with DNS which resulted into unknown host exception for
> one the machines in our env and caused Solr to throw the above exception
>
> Eric, I have the Solr configured using service installation script and
> the ZK_HOST entry in
> solr.in.sh="server1:2181,server2:2181,server3:2181/collection"
> and after removing the server1 from above, was able to start Solr otherwise
> it was throwing above exception.
>
> Thnx
>
>
> On Mon, Jul 23, 2018 at 4:20 PM, Erick Erickson <er...@gmail.com>
> wrote:
>
> > And how do you start Solr? Do you use the entire 3-node ensemble address?
> >
> > On Mon, Jul 23, 2018 at 12:55 PM, Michael Braun <n3...@gmail.com>
> wrote:
> > > Per the exception, this looks like a network / DNS resolution issue,
> > > independent of Solr and Zookeeper code:
> > >
> > > Caused by: org.apache.solr.common.SolrException:
> > > java.net.UnknownHostException: ditsearch001.es.com: Name or service
> not
> > > known
> > >
> > > Is this address actually resolvable at the time?
> > >
> > > On Mon, Jul 23, 2018 at 3:46 PM, Susheel Kumar <su...@gmail.com>
> > > wrote:
> > >
> > >> In usual circumstances when one Zookeeper goes down while others 2 are
> > up,
> > >> Solr continues to operate but when one of the ZK machine was not
> > reachable
> > >> with ping returning below results, Solr count't starts. See stack
> trace
> > >> below
> > >>
> > >> ping: cannot resolve ditsearch001.es.com: Unknown host
> > >>
> > >>
> > >> Setup: Solr 6.6.2 and Zookeeper 3.4.10
> > >>
> > >> I had to remove this server name from the ZK_HOST list (solr.in.sh)
> in
> > >> order to get Solr started. Ideally whatever issue is there as far as
> > >> majority is there, Solr should get started.
> > >>
> > >> Has any one noticed this issue?
> > >>
> > >> Thnx
> > >>
> > >> 2018-07-23 15:30:47.218 INFO (main) [ ] o.e.j.s.Server
> > >> jetty-9.3.14.v20161028
> > >>
> > >> 2018-07-23 15:30:47.817 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> > ___
> > >> _ Welcome to Apache Solr‚Ñ¢ version 6.6.2
> > >>
> > >> 2018-07-23 15:30:47.829 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> /
> > __|
> > >> ___| |_ _ Starting in cloud mode on port 8080
> > >>
> > >> 2018-07-23 15:30:47.830 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> > \__
> > >> \/ _ \ | '_| Install dir: /opt/solr
> > >>
> > >> 2018-07-23 15:30:47.861 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> > >> |___/\___/_|_| Start time: 2018-07-23T15:30:47.832Z
> > >>
> > >> 2018-07-23 15:30:47.863 INFO (main) [ ] o.a.s.s.StartupLoggingUtils
> > >> Property solr.log.muteconsole given. Muting ConsoleAppender named
> > CONSOLE
> > >>
> > >> 2018-07-23 15:30:47.929 INFO (main) [ ] o.a.s.c.SolrResourceLoader
> > Using
> > >> system property solr.solr.home: /app/solr/data
> > >>
> > >> 2018-07-23 15:30:48.037 ERROR (main) [ ] o.a.s.s.SolrDispatchFilter
> > Could
> > >> not start Solr. Check solr/home property and the logs
> > >>
> > >> 2018-07-23 15:30:48.235 ERROR (main) [ ] o.a.s.c.SolrCore
> > >> null:org.apache.solr.common.SolrException: Error occurred while
> loading
> > >> solr.xml from zookeeper
> > >>
> > >> at
> > >> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> > >> SolrDispatchFilter.java:270)
> > >>
> > >> at
> > >> org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(
> > >> SolrDispatchFilter.java:242)
> > >>
> > >> at
> > >> org.apache.solr.servlet.SolrDispatchFilter.init(
> > >> SolrDispatchFilter.java:173)
> > >>
> > >> at
> > >> org.eclipse.jetty.servlet.FilterHolder.initialize(
> > FilterHolder.java:137)
> > >>
> > >> at
> > >> org.eclipse.jetty.servlet.ServletHandler.initialize(
> > >> ServletHandler.java:873)
> > >>
> > >> at
> > >> org.eclipse.jetty.servlet.ServletContextHandler.startContext(
> > >> ServletContextHandler.java:349)
> > >>
> > >> at
> > >> org.eclipse.jetty.webapp.WebAppContext.startWebapp(
> > >> WebAppContext.java:1404)
> > >>
> > >> at
> > >> org.eclipse.jetty.webapp.WebAppContext.startContext(
> > >> WebAppContext.java:1366)
> > >>
> > >> at
> > >> org.eclipse.jetty.server.handler.ContextHandler.
> > >> doStart(ContextHandler.java:778)
> > >>
> > >> at
> > >> org.eclipse.jetty.servlet.ServletContextHandler.doStart(
> > >> ServletContextHandler.java:262)
> > >>
> > >> at
> > >> org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:520)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > >> start(AbstractLifeCycle.java:68)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.bindings.StandardStarter.processBinding(
> > >> StandardStarter.java:41)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.AppLifeCycle.runBindings(
> > AppLifeCycle.java:188)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.DeploymentManager.requestAppGoal(
> > >> DeploymentManager.java:499)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.DeploymentManager.addApp(
> > >> DeploymentManager.java:147)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> > >> fileAdded(ScanningAppProvider.java:180)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.providers.WebAppProvider.
> > >> fileAdded(WebAppProvider.java:458)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.providers.ScanningAppProvider$1.fileAdded(
> > >> ScanningAppProvider.java:64)
> > >>
> > >> at org.eclipse.jetty.util.Scanner.reportAddition(
> > Scanner.java:610)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.Scanner.reportDifferences(Scanner.java:529)
> > >>
> > >> at org.eclipse.jetty.util.Scanner.scan(Scanner.java:392)
> > >>
> > >> at org.eclipse.jetty.util.Scanner.doStart(Scanner.java:313)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > >> start(AbstractLifeCycle.java:68)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> > >> doStart(ScanningAppProvider.java:150)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > >> start(AbstractLifeCycle.java:68)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.DeploymentManager.startAppProvider(
> > >> DeploymentManager.java:561)
> > >>
> > >> at
> > >> org.eclipse.jetty.deploy.DeploymentManager.doStart(
> > >> DeploymentManager.java:236)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > >> start(AbstractLifeCycle.java:68)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.component.ContainerLifeCycle.
> > >> start(ContainerLifeCycle.java:131)
> > >>
> > >> at org.eclipse.jetty.server.Server.start(Server.java:422)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.component.ContainerLifeCycle.
> > >> doStart(ContainerLifeCycle.java:113)
> > >>
> > >> at
> > >> org.eclipse.jetty.server.handler.AbstractHandler.
> > >> doStart(AbstractHandler.java:61)
> > >>
> > >> at org.eclipse.jetty.server.Server.doStart(Server.java:389)
> > >>
> > >> at
> > >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> > >> start(AbstractLifeCycle.java:68)
> > >>
> > >> at
> > >> org.eclipse.jetty.xml.XmlConfiguration$1.run(
> > XmlConfiguration.java:1516)
> > >>
> > >> at java.security.AccessController.doPrivileged(Native Method)
> > >>
> > >> at
> > >>
> org.eclipse.jetty.xml.XmlConfiguration.main(XmlConfiguration.java:1441)
> > >>
> > >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > >>
> > >> at
> > >> sun.reflect.NativeMethodAccessorImpl.invoke(
> > NativeMethodAccessorImpl.java:
> > >> 62)
> > >>
> > >> at
> > >> sun.reflect.DelegatingMethodAccessorImpl.invoke(
> > >> DelegatingMethodAccessorImpl.java:43)
> > >>
> > >> at java.lang.reflect.Method.invoke(Method.java:498)
> > >>
> > >> at org.eclipse.jetty.start.Main.invokeMain(Main.java:214)
> > >>
> > >> at org.eclipse.jetty.start.Main.start(Main.java:457)
> > >>
> > >> at org.eclipse.jetty.start.Main.main(Main.java:75)
> > >>
> > >> Caused by: org.apache.solr.common.SolrException:
> > >> java.net.UnknownHostException: ditsearch001.es.com: Name or service
> not
> > >> known
> > >>
> > >> at
> > >>
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
> > >>
> > >> at
> > >>
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
> > >>
> > >> at
> > >>
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
> > >>
> > >> at
> > >> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
> > >>
> > >> at
> > >> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> > >> SolrDispatchFilter.java:263)
> > >>
> > >> ... 44 more
> > >>
> > >> Caused by: java.net.UnknownHostException: ditsearch001.es.com: Name
> or
> > >> service not known
> > >>
> > >> at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
> > >>
> > >> at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.
> > java:928)
> > >>
> > >> at
> > >>
> java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
> > >>
> > >> at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
> > >>
> > >> at java.net.InetAddress.getAllByName(InetAddress.java:1192)
> > >>
> > >> at java.net.InetAddress.getAllByName(InetAddress.java:1126)
> > >>
> > >> at
> > >> org.apache.zookeeper.client.StaticHostProvider.<init>(
> > >> StaticHostProvider.java:61)
> > >>
> > >> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:445)
> > >>
> > >> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
> > >>
> > >> at
> > >> org.apache.solr.common.cloud.SolrZooKeeper.<init>(
> > SolrZooKeeper.java:43)
> > >>
> > >> at
> > >> org.apache.solr.common.cloud.ZkClientConnectionStrategy.
> > >> createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
> > >>
> > >> at
> > >> org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(
> > >> DefaultConnectionStrategy.java:37)
> > >>
> > >> at
> > >>
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
> > >>
> > >> ... 48 more
> > >>
> >
>
--
Regards,
Shalin Shekhar Mangar.
Re: Solr fails even ZK quorum has majority
Posted by Susheel Kumar <su...@gmail.com>.
Something messed up with DNS which resulted into unknown host exception for
one the machines in our env and caused Solr to throw the above exception
Eric, I have the Solr configured using service installation script and
the ZK_HOST entry in
solr.in.sh="server1:2181,server2:2181,server3:2181/collection"
and after removing the server1 from above, was able to start Solr otherwise
it was throwing above exception.
Thnx
On Mon, Jul 23, 2018 at 4:20 PM, Erick Erickson <er...@gmail.com>
wrote:
> And how do you start Solr? Do you use the entire 3-node ensemble address?
>
> On Mon, Jul 23, 2018 at 12:55 PM, Michael Braun <n3...@gmail.com> wrote:
> > Per the exception, this looks like a network / DNS resolution issue,
> > independent of Solr and Zookeeper code:
> >
> > Caused by: org.apache.solr.common.SolrException:
> > java.net.UnknownHostException: ditsearch001.es.com: Name or service not
> > known
> >
> > Is this address actually resolvable at the time?
> >
> > On Mon, Jul 23, 2018 at 3:46 PM, Susheel Kumar <su...@gmail.com>
> > wrote:
> >
> >> In usual circumstances when one Zookeeper goes down while others 2 are
> up,
> >> Solr continues to operate but when one of the ZK machine was not
> reachable
> >> with ping returning below results, Solr count't starts. See stack trace
> >> below
> >>
> >> ping: cannot resolve ditsearch001.es.com: Unknown host
> >>
> >>
> >> Setup: Solr 6.6.2 and Zookeeper 3.4.10
> >>
> >> I had to remove this server name from the ZK_HOST list (solr.in.sh) in
> >> order to get Solr started. Ideally whatever issue is there as far as
> >> majority is there, Solr should get started.
> >>
> >> Has any one noticed this issue?
> >>
> >> Thnx
> >>
> >> 2018-07-23 15:30:47.218 INFO (main) [ ] o.e.j.s.Server
> >> jetty-9.3.14.v20161028
> >>
> >> 2018-07-23 15:30:47.817 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> ___
> >> _ Welcome to Apache Solr‚Ñ¢ version 6.6.2
> >>
> >> 2018-07-23 15:30:47.829 INFO (main) [ ] o.a.s.s.SolrDispatchFilter /
> __|
> >> ___| |_ _ Starting in cloud mode on port 8080
> >>
> >> 2018-07-23 15:30:47.830 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> \__
> >> \/ _ \ | '_| Install dir: /opt/solr
> >>
> >> 2018-07-23 15:30:47.861 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> >> |___/\___/_|_| Start time: 2018-07-23T15:30:47.832Z
> >>
> >> 2018-07-23 15:30:47.863 INFO (main) [ ] o.a.s.s.StartupLoggingUtils
> >> Property solr.log.muteconsole given. Muting ConsoleAppender named
> CONSOLE
> >>
> >> 2018-07-23 15:30:47.929 INFO (main) [ ] o.a.s.c.SolrResourceLoader
> Using
> >> system property solr.solr.home: /app/solr/data
> >>
> >> 2018-07-23 15:30:48.037 ERROR (main) [ ] o.a.s.s.SolrDispatchFilter
> Could
> >> not start Solr. Check solr/home property and the logs
> >>
> >> 2018-07-23 15:30:48.235 ERROR (main) [ ] o.a.s.c.SolrCore
> >> null:org.apache.solr.common.SolrException: Error occurred while loading
> >> solr.xml from zookeeper
> >>
> >> at
> >> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> >> SolrDispatchFilter.java:270)
> >>
> >> at
> >> org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(
> >> SolrDispatchFilter.java:242)
> >>
> >> at
> >> org.apache.solr.servlet.SolrDispatchFilter.init(
> >> SolrDispatchFilter.java:173)
> >>
> >> at
> >> org.eclipse.jetty.servlet.FilterHolder.initialize(
> FilterHolder.java:137)
> >>
> >> at
> >> org.eclipse.jetty.servlet.ServletHandler.initialize(
> >> ServletHandler.java:873)
> >>
> >> at
> >> org.eclipse.jetty.servlet.ServletContextHandler.startContext(
> >> ServletContextHandler.java:349)
> >>
> >> at
> >> org.eclipse.jetty.webapp.WebAppContext.startWebapp(
> >> WebAppContext.java:1404)
> >>
> >> at
> >> org.eclipse.jetty.webapp.WebAppContext.startContext(
> >> WebAppContext.java:1366)
> >>
> >> at
> >> org.eclipse.jetty.server.handler.ContextHandler.
> >> doStart(ContextHandler.java:778)
> >>
> >> at
> >> org.eclipse.jetty.servlet.ServletContextHandler.doStart(
> >> ServletContextHandler.java:262)
> >>
> >> at
> >> org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:520)
> >>
> >> at
> >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> >> start(AbstractLifeCycle.java:68)
> >>
> >> at
> >> org.eclipse.jetty.deploy.bindings.StandardStarter.processBinding(
> >> StandardStarter.java:41)
> >>
> >> at
> >> org.eclipse.jetty.deploy.AppLifeCycle.runBindings(
> AppLifeCycle.java:188)
> >>
> >> at
> >> org.eclipse.jetty.deploy.DeploymentManager.requestAppGoal(
> >> DeploymentManager.java:499)
> >>
> >> at
> >> org.eclipse.jetty.deploy.DeploymentManager.addApp(
> >> DeploymentManager.java:147)
> >>
> >> at
> >> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> >> fileAdded(ScanningAppProvider.java:180)
> >>
> >> at
> >> org.eclipse.jetty.deploy.providers.WebAppProvider.
> >> fileAdded(WebAppProvider.java:458)
> >>
> >> at
> >> org.eclipse.jetty.deploy.providers.ScanningAppProvider$1.fileAdded(
> >> ScanningAppProvider.java:64)
> >>
> >> at org.eclipse.jetty.util.Scanner.reportAddition(
> Scanner.java:610)
> >>
> >> at
> >> org.eclipse.jetty.util.Scanner.reportDifferences(Scanner.java:529)
> >>
> >> at org.eclipse.jetty.util.Scanner.scan(Scanner.java:392)
> >>
> >> at org.eclipse.jetty.util.Scanner.doStart(Scanner.java:313)
> >>
> >> at
> >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> >> start(AbstractLifeCycle.java:68)
> >>
> >> at
> >> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> >> doStart(ScanningAppProvider.java:150)
> >>
> >> at
> >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> >> start(AbstractLifeCycle.java:68)
> >>
> >> at
> >> org.eclipse.jetty.deploy.DeploymentManager.startAppProvider(
> >> DeploymentManager.java:561)
> >>
> >> at
> >> org.eclipse.jetty.deploy.DeploymentManager.doStart(
> >> DeploymentManager.java:236)
> >>
> >> at
> >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> >> start(AbstractLifeCycle.java:68)
> >>
> >> at
> >> org.eclipse.jetty.util.component.ContainerLifeCycle.
> >> start(ContainerLifeCycle.java:131)
> >>
> >> at org.eclipse.jetty.server.Server.start(Server.java:422)
> >>
> >> at
> >> org.eclipse.jetty.util.component.ContainerLifeCycle.
> >> doStart(ContainerLifeCycle.java:113)
> >>
> >> at
> >> org.eclipse.jetty.server.handler.AbstractHandler.
> >> doStart(AbstractHandler.java:61)
> >>
> >> at org.eclipse.jetty.server.Server.doStart(Server.java:389)
> >>
> >> at
> >> org.eclipse.jetty.util.component.AbstractLifeCycle.
> >> start(AbstractLifeCycle.java:68)
> >>
> >> at
> >> org.eclipse.jetty.xml.XmlConfiguration$1.run(
> XmlConfiguration.java:1516)
> >>
> >> at java.security.AccessController.doPrivileged(Native Method)
> >>
> >> at
> >> org.eclipse.jetty.xml.XmlConfiguration.main(XmlConfiguration.java:1441)
> >>
> >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >>
> >> at
> >> sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:
> >> 62)
> >>
> >> at
> >> sun.reflect.DelegatingMethodAccessorImpl.invoke(
> >> DelegatingMethodAccessorImpl.java:43)
> >>
> >> at java.lang.reflect.Method.invoke(Method.java:498)
> >>
> >> at org.eclipse.jetty.start.Main.invokeMain(Main.java:214)
> >>
> >> at org.eclipse.jetty.start.Main.start(Main.java:457)
> >>
> >> at org.eclipse.jetty.start.Main.main(Main.java:75)
> >>
> >> Caused by: org.apache.solr.common.SolrException:
> >> java.net.UnknownHostException: ditsearch001.es.com: Name or service not
> >> known
> >>
> >> at
> >> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
> >>
> >> at
> >> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
> >>
> >> at
> >> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
> >>
> >> at
> >> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
> >>
> >> at
> >> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> >> SolrDispatchFilter.java:263)
> >>
> >> ... 44 more
> >>
> >> Caused by: java.net.UnknownHostException: ditsearch001.es.com: Name or
> >> service not known
> >>
> >> at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
> >>
> >> at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.
> java:928)
> >>
> >> at
> >> java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
> >>
> >> at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
> >>
> >> at java.net.InetAddress.getAllByName(InetAddress.java:1192)
> >>
> >> at java.net.InetAddress.getAllByName(InetAddress.java:1126)
> >>
> >> at
> >> org.apache.zookeeper.client.StaticHostProvider.<init>(
> >> StaticHostProvider.java:61)
> >>
> >> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:445)
> >>
> >> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
> >>
> >> at
> >> org.apache.solr.common.cloud.SolrZooKeeper.<init>(
> SolrZooKeeper.java:43)
> >>
> >> at
> >> org.apache.solr.common.cloud.ZkClientConnectionStrategy.
> >> createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
> >>
> >> at
> >> org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(
> >> DefaultConnectionStrategy.java:37)
> >>
> >> at
> >> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
> >>
> >> ... 48 more
> >>
>
Re: Solr fails even ZK quorum has majority
Posted by Erick Erickson <er...@gmail.com>.
And how do you start Solr? Do you use the entire 3-node ensemble address?
On Mon, Jul 23, 2018 at 12:55 PM, Michael Braun <n3...@gmail.com> wrote:
> Per the exception, this looks like a network / DNS resolution issue,
> independent of Solr and Zookeeper code:
>
> Caused by: org.apache.solr.common.SolrException:
> java.net.UnknownHostException: ditsearch001.es.com: Name or service not
> known
>
> Is this address actually resolvable at the time?
>
> On Mon, Jul 23, 2018 at 3:46 PM, Susheel Kumar <su...@gmail.com>
> wrote:
>
>> In usual circumstances when one Zookeeper goes down while others 2 are up,
>> Solr continues to operate but when one of the ZK machine was not reachable
>> with ping returning below results, Solr count't starts. See stack trace
>> below
>>
>> ping: cannot resolve ditsearch001.es.com: Unknown host
>>
>>
>> Setup: Solr 6.6.2 and Zookeeper 3.4.10
>>
>> I had to remove this server name from the ZK_HOST list (solr.in.sh) in
>> order to get Solr started. Ideally whatever issue is there as far as
>> majority is there, Solr should get started.
>>
>> Has any one noticed this issue?
>>
>> Thnx
>>
>> 2018-07-23 15:30:47.218 INFO (main) [ ] o.e.j.s.Server
>> jetty-9.3.14.v20161028
>>
>> 2018-07-23 15:30:47.817 INFO (main) [ ] o.a.s.s.SolrDispatchFilter ___
>> _ Welcome to Apache Solr‚Ñ¢ version 6.6.2
>>
>> 2018-07-23 15:30:47.829 INFO (main) [ ] o.a.s.s.SolrDispatchFilter / __|
>> ___| |_ _ Starting in cloud mode on port 8080
>>
>> 2018-07-23 15:30:47.830 INFO (main) [ ] o.a.s.s.SolrDispatchFilter \__
>> \/ _ \ | '_| Install dir: /opt/solr
>>
>> 2018-07-23 15:30:47.861 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
>> |___/\___/_|_| Start time: 2018-07-23T15:30:47.832Z
>>
>> 2018-07-23 15:30:47.863 INFO (main) [ ] o.a.s.s.StartupLoggingUtils
>> Property solr.log.muteconsole given. Muting ConsoleAppender named CONSOLE
>>
>> 2018-07-23 15:30:47.929 INFO (main) [ ] o.a.s.c.SolrResourceLoader Using
>> system property solr.solr.home: /app/solr/data
>>
>> 2018-07-23 15:30:48.037 ERROR (main) [ ] o.a.s.s.SolrDispatchFilter Could
>> not start Solr. Check solr/home property and the logs
>>
>> 2018-07-23 15:30:48.235 ERROR (main) [ ] o.a.s.c.SolrCore
>> null:org.apache.solr.common.SolrException: Error occurred while loading
>> solr.xml from zookeeper
>>
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
>> SolrDispatchFilter.java:270)
>>
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(
>> SolrDispatchFilter.java:242)
>>
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.init(
>> SolrDispatchFilter.java:173)
>>
>> at
>> org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
>>
>> at
>> org.eclipse.jetty.servlet.ServletHandler.initialize(
>> ServletHandler.java:873)
>>
>> at
>> org.eclipse.jetty.servlet.ServletContextHandler.startContext(
>> ServletContextHandler.java:349)
>>
>> at
>> org.eclipse.jetty.webapp.WebAppContext.startWebapp(
>> WebAppContext.java:1404)
>>
>> at
>> org.eclipse.jetty.webapp.WebAppContext.startContext(
>> WebAppContext.java:1366)
>>
>> at
>> org.eclipse.jetty.server.handler.ContextHandler.
>> doStart(ContextHandler.java:778)
>>
>> at
>> org.eclipse.jetty.servlet.ServletContextHandler.doStart(
>> ServletContextHandler.java:262)
>>
>> at
>> org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:520)
>>
>> at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.
>> start(AbstractLifeCycle.java:68)
>>
>> at
>> org.eclipse.jetty.deploy.bindings.StandardStarter.processBinding(
>> StandardStarter.java:41)
>>
>> at
>> org.eclipse.jetty.deploy.AppLifeCycle.runBindings(AppLifeCycle.java:188)
>>
>> at
>> org.eclipse.jetty.deploy.DeploymentManager.requestAppGoal(
>> DeploymentManager.java:499)
>>
>> at
>> org.eclipse.jetty.deploy.DeploymentManager.addApp(
>> DeploymentManager.java:147)
>>
>> at
>> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
>> fileAdded(ScanningAppProvider.java:180)
>>
>> at
>> org.eclipse.jetty.deploy.providers.WebAppProvider.
>> fileAdded(WebAppProvider.java:458)
>>
>> at
>> org.eclipse.jetty.deploy.providers.ScanningAppProvider$1.fileAdded(
>> ScanningAppProvider.java:64)
>>
>> at org.eclipse.jetty.util.Scanner.reportAddition(Scanner.java:610)
>>
>> at
>> org.eclipse.jetty.util.Scanner.reportDifferences(Scanner.java:529)
>>
>> at org.eclipse.jetty.util.Scanner.scan(Scanner.java:392)
>>
>> at org.eclipse.jetty.util.Scanner.doStart(Scanner.java:313)
>>
>> at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.
>> start(AbstractLifeCycle.java:68)
>>
>> at
>> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
>> doStart(ScanningAppProvider.java:150)
>>
>> at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.
>> start(AbstractLifeCycle.java:68)
>>
>> at
>> org.eclipse.jetty.deploy.DeploymentManager.startAppProvider(
>> DeploymentManager.java:561)
>>
>> at
>> org.eclipse.jetty.deploy.DeploymentManager.doStart(
>> DeploymentManager.java:236)
>>
>> at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.
>> start(AbstractLifeCycle.java:68)
>>
>> at
>> org.eclipse.jetty.util.component.ContainerLifeCycle.
>> start(ContainerLifeCycle.java:131)
>>
>> at org.eclipse.jetty.server.Server.start(Server.java:422)
>>
>> at
>> org.eclipse.jetty.util.component.ContainerLifeCycle.
>> doStart(ContainerLifeCycle.java:113)
>>
>> at
>> org.eclipse.jetty.server.handler.AbstractHandler.
>> doStart(AbstractHandler.java:61)
>>
>> at org.eclipse.jetty.server.Server.doStart(Server.java:389)
>>
>> at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.
>> start(AbstractLifeCycle.java:68)
>>
>> at
>> org.eclipse.jetty.xml.XmlConfiguration$1.run(XmlConfiguration.java:1516)
>>
>> at java.security.AccessController.doPrivileged(Native Method)
>>
>> at
>> org.eclipse.jetty.xml.XmlConfiguration.main(XmlConfiguration.java:1441)
>>
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>
>> at
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:
>> 62)
>>
>> at
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(
>> DelegatingMethodAccessorImpl.java:43)
>>
>> at java.lang.reflect.Method.invoke(Method.java:498)
>>
>> at org.eclipse.jetty.start.Main.invokeMain(Main.java:214)
>>
>> at org.eclipse.jetty.start.Main.start(Main.java:457)
>>
>> at org.eclipse.jetty.start.Main.main(Main.java:75)
>>
>> Caused by: org.apache.solr.common.SolrException:
>> java.net.UnknownHostException: ditsearch001.es.com: Name or service not
>> known
>>
>> at
>> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
>>
>> at
>> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
>>
>> at
>> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
>>
>> at
>> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
>>
>> at
>> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
>> SolrDispatchFilter.java:263)
>>
>> ... 44 more
>>
>> Caused by: java.net.UnknownHostException: ditsearch001.es.com: Name or
>> service not known
>>
>> at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
>>
>> at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
>>
>> at
>> java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
>>
>> at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
>>
>> at java.net.InetAddress.getAllByName(InetAddress.java:1192)
>>
>> at java.net.InetAddress.getAllByName(InetAddress.java:1126)
>>
>> at
>> org.apache.zookeeper.client.StaticHostProvider.<init>(
>> StaticHostProvider.java:61)
>>
>> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:445)
>>
>> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
>>
>> at
>> org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
>>
>> at
>> org.apache.solr.common.cloud.ZkClientConnectionStrategy.
>> createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
>>
>> at
>> org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(
>> DefaultConnectionStrategy.java:37)
>>
>> at
>> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
>>
>> ... 48 more
>>
Re: Solr fails even ZK quorum has majority
Posted by Michael Braun <n3...@gmail.com>.
Per the exception, this looks like a network / DNS resolution issue,
independent of Solr and Zookeeper code:
Caused by: org.apache.solr.common.SolrException:
java.net.UnknownHostException: ditsearch001.es.com: Name or service not
known
Is this address actually resolvable at the time?
On Mon, Jul 23, 2018 at 3:46 PM, Susheel Kumar <su...@gmail.com>
wrote:
> In usual circumstances when one Zookeeper goes down while others 2 are up,
> Solr continues to operate but when one of the ZK machine was not reachable
> with ping returning below results, Solr count't starts. See stack trace
> below
>
> ping: cannot resolve ditsearch001.es.com: Unknown host
>
>
> Setup: Solr 6.6.2 and Zookeeper 3.4.10
>
> I had to remove this server name from the ZK_HOST list (solr.in.sh) in
> order to get Solr started. Ideally whatever issue is there as far as
> majority is there, Solr should get started.
>
> Has any one noticed this issue?
>
> Thnx
>
> 2018-07-23 15:30:47.218 INFO (main) [ ] o.e.j.s.Server
> jetty-9.3.14.v20161028
>
> 2018-07-23 15:30:47.817 INFO (main) [ ] o.a.s.s.SolrDispatchFilter ___
> _ Welcome to Apache Solr‚Ñ¢ version 6.6.2
>
> 2018-07-23 15:30:47.829 INFO (main) [ ] o.a.s.s.SolrDispatchFilter / __|
> ___| |_ _ Starting in cloud mode on port 8080
>
> 2018-07-23 15:30:47.830 INFO (main) [ ] o.a.s.s.SolrDispatchFilter \__
> \/ _ \ | '_| Install dir: /opt/solr
>
> 2018-07-23 15:30:47.861 INFO (main) [ ] o.a.s.s.SolrDispatchFilter
> |___/\___/_|_| Start time: 2018-07-23T15:30:47.832Z
>
> 2018-07-23 15:30:47.863 INFO (main) [ ] o.a.s.s.StartupLoggingUtils
> Property solr.log.muteconsole given. Muting ConsoleAppender named CONSOLE
>
> 2018-07-23 15:30:47.929 INFO (main) [ ] o.a.s.c.SolrResourceLoader Using
> system property solr.solr.home: /app/solr/data
>
> 2018-07-23 15:30:48.037 ERROR (main) [ ] o.a.s.s.SolrDispatchFilter Could
> not start Solr. Check solr/home property and the logs
>
> 2018-07-23 15:30:48.235 ERROR (main) [ ] o.a.s.c.SolrCore
> null:org.apache.solr.common.SolrException: Error occurred while loading
> solr.xml from zookeeper
>
> at
> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> SolrDispatchFilter.java:270)
>
> at
> org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(
> SolrDispatchFilter.java:242)
>
> at
> org.apache.solr.servlet.SolrDispatchFilter.init(
> SolrDispatchFilter.java:173)
>
> at
> org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:137)
>
> at
> org.eclipse.jetty.servlet.ServletHandler.initialize(
> ServletHandler.java:873)
>
> at
> org.eclipse.jetty.servlet.ServletContextHandler.startContext(
> ServletContextHandler.java:349)
>
> at
> org.eclipse.jetty.webapp.WebAppContext.startWebapp(
> WebAppContext.java:1404)
>
> at
> org.eclipse.jetty.webapp.WebAppContext.startContext(
> WebAppContext.java:1366)
>
> at
> org.eclipse.jetty.server.handler.ContextHandler.
> doStart(ContextHandler.java:778)
>
> at
> org.eclipse.jetty.servlet.ServletContextHandler.doStart(
> ServletContextHandler.java:262)
>
> at
> org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:520)
>
> at
> org.eclipse.jetty.util.component.AbstractLifeCycle.
> start(AbstractLifeCycle.java:68)
>
> at
> org.eclipse.jetty.deploy.bindings.StandardStarter.processBinding(
> StandardStarter.java:41)
>
> at
> org.eclipse.jetty.deploy.AppLifeCycle.runBindings(AppLifeCycle.java:188)
>
> at
> org.eclipse.jetty.deploy.DeploymentManager.requestAppGoal(
> DeploymentManager.java:499)
>
> at
> org.eclipse.jetty.deploy.DeploymentManager.addApp(
> DeploymentManager.java:147)
>
> at
> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> fileAdded(ScanningAppProvider.java:180)
>
> at
> org.eclipse.jetty.deploy.providers.WebAppProvider.
> fileAdded(WebAppProvider.java:458)
>
> at
> org.eclipse.jetty.deploy.providers.ScanningAppProvider$1.fileAdded(
> ScanningAppProvider.java:64)
>
> at org.eclipse.jetty.util.Scanner.reportAddition(Scanner.java:610)
>
> at
> org.eclipse.jetty.util.Scanner.reportDifferences(Scanner.java:529)
>
> at org.eclipse.jetty.util.Scanner.scan(Scanner.java:392)
>
> at org.eclipse.jetty.util.Scanner.doStart(Scanner.java:313)
>
> at
> org.eclipse.jetty.util.component.AbstractLifeCycle.
> start(AbstractLifeCycle.java:68)
>
> at
> org.eclipse.jetty.deploy.providers.ScanningAppProvider.
> doStart(ScanningAppProvider.java:150)
>
> at
> org.eclipse.jetty.util.component.AbstractLifeCycle.
> start(AbstractLifeCycle.java:68)
>
> at
> org.eclipse.jetty.deploy.DeploymentManager.startAppProvider(
> DeploymentManager.java:561)
>
> at
> org.eclipse.jetty.deploy.DeploymentManager.doStart(
> DeploymentManager.java:236)
>
> at
> org.eclipse.jetty.util.component.AbstractLifeCycle.
> start(AbstractLifeCycle.java:68)
>
> at
> org.eclipse.jetty.util.component.ContainerLifeCycle.
> start(ContainerLifeCycle.java:131)
>
> at org.eclipse.jetty.server.Server.start(Server.java:422)
>
> at
> org.eclipse.jetty.util.component.ContainerLifeCycle.
> doStart(ContainerLifeCycle.java:113)
>
> at
> org.eclipse.jetty.server.handler.AbstractHandler.
> doStart(AbstractHandler.java:61)
>
> at org.eclipse.jetty.server.Server.doStart(Server.java:389)
>
> at
> org.eclipse.jetty.util.component.AbstractLifeCycle.
> start(AbstractLifeCycle.java:68)
>
> at
> org.eclipse.jetty.xml.XmlConfiguration$1.run(XmlConfiguration.java:1516)
>
> at java.security.AccessController.doPrivileged(Native Method)
>
> at
> org.eclipse.jetty.xml.XmlConfiguration.main(XmlConfiguration.java:1441)
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:
> 62)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> at org.eclipse.jetty.start.Main.invokeMain(Main.java:214)
>
> at org.eclipse.jetty.start.Main.start(Main.java:457)
>
> at org.eclipse.jetty.start.Main.main(Main.java:75)
>
> Caused by: org.apache.solr.common.SolrException:
> java.net.UnknownHostException: ditsearch001.es.com: Name or service not
> known
>
> at
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
>
> at
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
>
> at
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
>
> at
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
>
> at
> org.apache.solr.servlet.SolrDispatchFilter.loadNodeConfig(
> SolrDispatchFilter.java:263)
>
> ... 44 more
>
> Caused by: java.net.UnknownHostException: ditsearch001.es.com: Name or
> service not known
>
> at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
>
> at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
>
> at
> java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
>
> at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
>
> at java.net.InetAddress.getAllByName(InetAddress.java:1192)
>
> at java.net.InetAddress.getAllByName(InetAddress.java:1126)
>
> at
> org.apache.zookeeper.client.StaticHostProvider.<init>(
> StaticHostProvider.java:61)
>
> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:445)
>
> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
>
> at
> org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
>
> at
> org.apache.solr.common.cloud.ZkClientConnectionStrategy.
> createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
>
> at
> org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(
> DefaultConnectionStrategy.java:37)
>
> at
> org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
>
> ... 48 more
>