You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@ignite.apache.org by rishi007bansod <ri...@gmail.com> on 2017/11/07 12:31:19 UTC
Unable to connect ignite pods in Kubernetes using Ip-finder
Hi,
I have initialized my pods as per steps in link
https://apacheignite.readme.io/docs/kubernetes-deployment
<https://apacheignite.readme.io/docs/kubernetes-deployment> . I have also
tried setting Kubernetes network to *1. flannel 2. weave net*. But I get
same error in both cases. Following are error logs :
*Flannel Error
*
[11:39:30,256][SEVERE][main][TcpDiscoverySpi] Failed to get registered
addresses from IP finder on start (retrying every 2000 ms).
class org.apache.ignite.spi.IgniteSpiException: Failed to retrieve Ignite
pods IP addresses.
at
org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:172)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.registeredAddresses(TcpDiscoverySpi.java:1613)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.resolvedAddresses(TcpDiscoverySpi.java:1562)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.sendJoinRequestMessage(ServerImpl.java:974)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.joinTopology(ServerImpl.java:837)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.spiStart(ServerImpl.java:351)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:1850)
at
org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:268)
at
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:685)
at
org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1626)
at
org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:924)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1799)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1602)
at
org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1042)
at
org.apache.ignite.internal.IgnitionEx.startConfigurations(IgnitionEx.java:964)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:850)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:749)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:619)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:589)
at org.apache.ignite.Ignition.start(Ignition.java:347)
at
org.apache.ignite.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:302)
Caused by: java.net.UnknownHostException:
kubernetes.default.svc.cluster.local
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:184)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
at
sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:173)
at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
at
sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:264)
at sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:367)
at
sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1138)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1032)
at
sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
at
sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1546)
at
sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
at
sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
at
org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:153)
... 20 more
*Weave Net Error
*
[12:14:59] Security status [authentication=off, tls/ssl=off]
[12:15:05,243][SEVERE][main][TcpDiscoverySpi] Failed to get registered
addresses from IP finder on start (retrying every 2000 ms).
class org.apache.ignite.spi.IgniteSpiException: Failed to retrieve Ignite
pods IP addresses.
at
org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:172)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.registeredAddresses(TcpDiscoverySpi.java:1613)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.resolvedAddresses(TcpDiscoverySpi.java:1562)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.sendJoinRequestMessage(ServerImpl.java:974)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.joinTopology(ServerImpl.java:837)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.spiStart(ServerImpl.java:351)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:1850)
at
org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:268)
at
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:685)
at
org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1626)
at
org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:924)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1799)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1602)
at
org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1042)
at
org.apache.ignite.internal.IgnitionEx.startConfigurations(IgnitionEx.java:964)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:850)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:749)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:619)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:589)
at org.apache.ignite.Ignition.start(Ignition.java:347)
at
org.apache.ignite.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:302)
Caused by: java.io.IOException: Server returned HTTP response code: 403 for
URL:
https://kubernetes.default.svc.cluster.local:443/api/v1/namespaces/default/endpoints/ignite
at
sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1876)
at
sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
at
sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
at
org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:153)
... 20 more
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by Denis Magda <dm...@apache.org>.
Hi Rishikesh,
Thanks for pointing out to this. I've updated the stateless deployment doc.
--
Denis
On Tue, Sep 11, 2018 at 7:06 AM rishi007bansod <ri...@gmail.com>
wrote:
> "serviceAccountName: ignite" should be present in Pod Deployment
> specification as mentioned by Anton in post
>
> https://stackoverflow.com/questions/49395481/how-to-setmasterurl-in-ignite-xml-config-for-kubernetes-ipfinder/49405879#49405879
> <
> https://stackoverflow.com/questions/49395481/how-to-setmasterurl-in-ignite-xml-config-for-kubernetes-ipfinder/49405879#49405879>
>
> . It is currently absent in
> https://apacheignite.readme.io/docs/stateless-deployment
> <https://apacheignite.readme.io/docs/stateless-deployment>
> "ignite-deployment.yaml" file
>
> Thanks,
> Rishikesh
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by rishi007bansod <ri...@gmail.com>.
"serviceAccountName: ignite" should be present in Pod Deployment
specification as mentioned by Anton in post
https://stackoverflow.com/questions/49395481/how-to-setmasterurl-in-ignite-xml-config-for-kubernetes-ipfinder/49405879#49405879
<https://stackoverflow.com/questions/49395481/how-to-setmasterurl-in-ignite-xml-config-for-kubernetes-ipfinder/49405879#49405879>
. It is currently absent in
https://apacheignite.readme.io/docs/stateless-deployment
<https://apacheignite.readme.io/docs/stateless-deployment>
"ignite-deployment.yaml" file
Thanks,
Rishikesh
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by Denis Magda <dm...@apache.org>.
Guys,
Thanks for reporting and sharing the resolution. We'll update Ignite
Kubernetes deployment doc:
https://issues.apache.org/jira/browse/IGNITE-8081
--
Denis
On Tue, Mar 27, 2018 at 10:10 AM, lukaszbyjos <lu...@gmail.com>
wrote:
> I have the same problem on GKE with 2.4. I found some helpful info there
> but
> I see ignite neet more permissions.
> I think this should be added to Ignite kubernetes deployment instruction.
>
> https://stackoverflow.com/a/49405879/2578137
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by lukaszbyjos <lu...@gmail.com>.
I have the same problem on GKE with 2.4. I found some helpful info there but
I see ignite neet more permissions.
I think this should be added to Ignite kubernetes deployment instruction.
https://stackoverflow.com/a/49405879/2578137
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by Ryan Samo <ry...@gm.com>.
I just ran into this same issue recently and it turns out that the
permissions given to the default ServiceAccount in Kubernetes Role-based
access control (RBAC) are not high enough to allow for the
TcpDiscoveryKubernetesIpFinder to talk to the kubernetes service at
"https://kubernetes.default.svc.cluster.local:443/api/v1/namespaces/default/endpoints/ignite"
which is why you get a 403 unauthorized exception. I found a work around in
the link below which grants the default ServiceAccount a ClusterRole of
"cluster-admin" in K8, then the Ignite PODs can communicate.
https://github.com/fluent/fluentd-kubernetes-daemonset/issues/14
<https://github.com/fluent/fluentd-kubernetes-daemonset/issues/14>
My question is, does the community have any documentation or knowledge in
the Ignite space for what permissions are required in Kubernetes in order
for an Ignite cluster to operate properly? It seems like granting
"cluster-admin" could be a bit risky for a production solution, especially
if you plan to have many Ignite clusters, each with their own K8 namespace
for example. I read through the Kubernetes Deployment documentation for
Ignite and did not see any reference to RBAC which was implemented in K8
v1.8. I suspect that maybe the Ignite documentation was written prior to
this release?
Thanks in advance for light you could shed on the subject.
Kubernetes v1.9.2
Ignite v2.3.0
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by Denis Magda <dm...@apache.org>.
The exception still says that the pod cannot connect to the master. If the connection were successful the master would with a list of all Ignite pods deployed. While now the master simply rejects IP finder’s request: Caused by: java.net.ConnectException: Connection refused (Connection
refused)
Please ensure the container the pod is running in can establish a connection with the master.
—
Denis
> On Nov 8, 2017, at 5:53 AM, rishi007bansod <ri...@gmail.com> wrote:
>
> Hi,
> I have used API method *TcpDiscoveryKubernetesIpFinder* and set Ip
> address explicitly as *https://192.168.120.92 using ipFinder.setMasterUrl *.
> But still I am unable to retrieve ip addresses of running ignite pods. I am
> using flannel network in kubernetes. Following is the error log I am getting
> :
>
> *[13:42:10,489][INFO][main][IgniteKernal]
>
>>>> __________ ________________
>>>> / _/ ___/ |/ / _/_ __/ __/
>>>> _/ // (7 7 // / / / / _/
>>>> /___/\___/_/|_/___/ /_/ /___/
>>>>
>>>> ver. 1.9.0#20170302-sha1:a8169d0a
>>>> 2017 Copyright(C) Apache Software Foundation
>>>>
>>>> Ignite documentation: http://ignite.apache.org
>
> [13:42:10,491][INFO][main][IgniteKernal] Config URL: n/a
> [13:42:10,491][INFO][main][IgniteKernal] Daemon mode: off
> [13:42:10,491][INFO][main][IgniteKernal] OS: Linux
> 3.10.0-514.21.2.el7.x86_64 amd64
> [13:42:10,491][INFO][main][IgniteKernal] OS user: root
> [13:42:10,496][INFO][main][IgniteKernal] PID: 7
> [13:42:10,496][INFO][main][IgniteKernal] Language runtime: Java Platform API
> Specification ver. 1.8
> [13:42:10,496][INFO][main][IgniteKernal] VM information: OpenJDK Runtime
> Environment 1.8.0_111-8u111-b14-2~bpo8+1-b14 Oracle Corporation OpenJDK
> 64-Bit Server VM 25.111-b14
> [13:42:10,499][INFO][main][IgniteKernal] VM total memory: 27.0GB
> [13:42:10,499][INFO][main][IgniteKernal] Remote Management [restart: off,
> REST: on, JMX (remote: off)]
> [13:42:10,499][INFO][main][IgniteKernal]
> IGNITE_HOME=/opt/ignite/apache-ignite-fabric-1.9.0-bin
> [13:42:10,500][INFO][main][IgniteKernal] VM arguments:
> [-DIGNITE_QUIET=false]
> [13:42:10,500][INFO][main][IgniteKernal] Configured caches
> ['ignite-marshaller-sys-cache', 'ignite-sys-cache',
> 'ignite-atomics-sys-cache']
> [13:42:10,508][INFO][main][IgniteKernal] 3-rd party licenses can be found
> at: /opt/ignite/apache-ignite-fabric-1.9.0-bin/libs/licenses
> [13:42:10,610][INFO][main][IgnitePluginProcessor] Configured plugins:
> [13:42:10,610][INFO][main][IgnitePluginProcessor] ^-- None
> [13:42:10,610][INFO][main][IgnitePluginProcessor]
> [13:42:10,703][INFO][main][TcpCommunicationSpi] Successfully bound
> communication NIO server to TCP port [port=47100, locHost=0.0.0.0/0.0.0.0,
> selectorsCnt=28, selectorSpins=0, pairedConn=false]
> [13:42:10,711][WARNING][main][TcpCommunicationSpi] Message queue limit is
> set to 0 which may lead to potential OOMEs when running cache operations in
> FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and
> receiver sides.
> [13:42:10,745][WARNING][main][NoopCheckpointSpi] Checkpoints are disabled
> (to enable configure any GridCheckpointSpi implementation)
> [13:42:10,787][WARNING][main][GridCollisionManager] Collision resolution is
> disabled (all jobs will be activated upon arrival).
> [13:42:10,792][WARNING][main][NoopSwapSpaceSpi] Swap space is disabled. To
> enable use FileSwapSpaceSpi.
> [13:42:10,794][INFO][main][IgniteKernal] Security status
> [authentication=off, tls/ssl=off]
> [13:42:11,254][INFO][main][GridTcpRestProtocol] Command protocol
> successfully started [name=TCP binary, host=0.0.0.0/0.0.0.0, port=11211]
> [13:42:11,303][INFO][main][IgniteKernal] Non-loopback local IPs: 172.17.0.4,
> fe80:0:0:0:42:acff:fe11:4%eth0
> [13:42:11,303][INFO][main][IgniteKernal] Enabled local MACs: 0242AC110004
> [13:42:11,344][INFO][main][TcpDiscoverySpi] Successfully bound to TCP port
> [port=47500, localHost=0.0.0.0/0.0.0.0,
> locNodeId=3c13c077-b9cc-4f00-91e2-57749c3fea32]
> [13:42:11,946][SEVERE][main][TcpDiscoverySpi] Failed to get registered
> addresses from IP finder on start (retrying every 2000 ms).
> class org.apache.ignite.spi.IgniteSpiException: Failed to retrieve Ignite
> pods IP addresses.
> at
> org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:172)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.registeredAddresses(TcpDiscoverySpi.java:1613)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.resolvedAddresses(TcpDiscoverySpi.java:1562)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.sendJoinRequestMessage(ServerImpl.java:974)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.joinTopology(ServerImpl.java:837)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.spiStart(ServerImpl.java:351)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:1850)
> at
> org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:268)
> at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:685)
> at
> org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1626)
> at
> org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:924)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1799)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1602)
> at
> org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1042)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:569)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:516)
> at org.apache.ignite.Ignition.start(Ignition.java:322)
> at xyz.CodeDeployment.main(CodeDeployment.java:85)
> Caused by: java.net.ConnectException: Connection refused (Connection
> refused)
> at java.net.PlainSocketImpl.socketConnect(Native Method)
> at
> java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
> at
> java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
> at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
> at java.net.Socket.connect(Socket.java:589)
> at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
> at
> sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:173)
> at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
> at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
> at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
> at
> sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:264)
> at sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:367)
> at
> sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
> at
> sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1138)
> at
> sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1032)
> at
> sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
> at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1546)
> at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
> at
> sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
> at
> org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:153)
> ... 17 more
> *
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by rishi007bansod <ri...@gmail.com>.
Hi,
I have used API method *TcpDiscoveryKubernetesIpFinder* and set Ip
address explicitly as *https://192.168.120.92 using ipFinder.setMasterUrl *.
But still I am unable to retrieve ip addresses of running ignite pods. I am
using flannel network in kubernetes. Following is the error log I am getting
:
*[13:42:10,489][INFO][main][IgniteKernal]
>>> __________ ________________
>>> / _/ ___/ |/ / _/_ __/ __/
>>> _/ // (7 7 // / / / / _/
>>> /___/\___/_/|_/___/ /_/ /___/
>>>
>>> ver. 1.9.0#20170302-sha1:a8169d0a
>>> 2017 Copyright(C) Apache Software Foundation
>>>
>>> Ignite documentation: http://ignite.apache.org
[13:42:10,491][INFO][main][IgniteKernal] Config URL: n/a
[13:42:10,491][INFO][main][IgniteKernal] Daemon mode: off
[13:42:10,491][INFO][main][IgniteKernal] OS: Linux
3.10.0-514.21.2.el7.x86_64 amd64
[13:42:10,491][INFO][main][IgniteKernal] OS user: root
[13:42:10,496][INFO][main][IgniteKernal] PID: 7
[13:42:10,496][INFO][main][IgniteKernal] Language runtime: Java Platform API
Specification ver. 1.8
[13:42:10,496][INFO][main][IgniteKernal] VM information: OpenJDK Runtime
Environment 1.8.0_111-8u111-b14-2~bpo8+1-b14 Oracle Corporation OpenJDK
64-Bit Server VM 25.111-b14
[13:42:10,499][INFO][main][IgniteKernal] VM total memory: 27.0GB
[13:42:10,499][INFO][main][IgniteKernal] Remote Management [restart: off,
REST: on, JMX (remote: off)]
[13:42:10,499][INFO][main][IgniteKernal]
IGNITE_HOME=/opt/ignite/apache-ignite-fabric-1.9.0-bin
[13:42:10,500][INFO][main][IgniteKernal] VM arguments:
[-DIGNITE_QUIET=false]
[13:42:10,500][INFO][main][IgniteKernal] Configured caches
['ignite-marshaller-sys-cache', 'ignite-sys-cache',
'ignite-atomics-sys-cache']
[13:42:10,508][INFO][main][IgniteKernal] 3-rd party licenses can be found
at: /opt/ignite/apache-ignite-fabric-1.9.0-bin/libs/licenses
[13:42:10,610][INFO][main][IgnitePluginProcessor] Configured plugins:
[13:42:10,610][INFO][main][IgnitePluginProcessor] ^-- None
[13:42:10,610][INFO][main][IgnitePluginProcessor]
[13:42:10,703][INFO][main][TcpCommunicationSpi] Successfully bound
communication NIO server to TCP port [port=47100, locHost=0.0.0.0/0.0.0.0,
selectorsCnt=28, selectorSpins=0, pairedConn=false]
[13:42:10,711][WARNING][main][TcpCommunicationSpi] Message queue limit is
set to 0 which may lead to potential OOMEs when running cache operations in
FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and
receiver sides.
[13:42:10,745][WARNING][main][NoopCheckpointSpi] Checkpoints are disabled
(to enable configure any GridCheckpointSpi implementation)
[13:42:10,787][WARNING][main][GridCollisionManager] Collision resolution is
disabled (all jobs will be activated upon arrival).
[13:42:10,792][WARNING][main][NoopSwapSpaceSpi] Swap space is disabled. To
enable use FileSwapSpaceSpi.
[13:42:10,794][INFO][main][IgniteKernal] Security status
[authentication=off, tls/ssl=off]
[13:42:11,254][INFO][main][GridTcpRestProtocol] Command protocol
successfully started [name=TCP binary, host=0.0.0.0/0.0.0.0, port=11211]
[13:42:11,303][INFO][main][IgniteKernal] Non-loopback local IPs: 172.17.0.4,
fe80:0:0:0:42:acff:fe11:4%eth0
[13:42:11,303][INFO][main][IgniteKernal] Enabled local MACs: 0242AC110004
[13:42:11,344][INFO][main][TcpDiscoverySpi] Successfully bound to TCP port
[port=47500, localHost=0.0.0.0/0.0.0.0,
locNodeId=3c13c077-b9cc-4f00-91e2-57749c3fea32]
[13:42:11,946][SEVERE][main][TcpDiscoverySpi] Failed to get registered
addresses from IP finder on start (retrying every 2000 ms).
class org.apache.ignite.spi.IgniteSpiException: Failed to retrieve Ignite
pods IP addresses.
at
org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:172)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.registeredAddresses(TcpDiscoverySpi.java:1613)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.resolvedAddresses(TcpDiscoverySpi.java:1562)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.sendJoinRequestMessage(ServerImpl.java:974)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.joinTopology(ServerImpl.java:837)
at
org.apache.ignite.spi.discovery.tcp.ServerImpl.spiStart(ServerImpl.java:351)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:1850)
at
org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:268)
at
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:685)
at
org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1626)
at
org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:924)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1799)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1602)
at
org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1042)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:569)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:516)
at org.apache.ignite.Ignition.start(Ignition.java:322)
at xyz.CodeDeployment.main(CodeDeployment.java:85)
Caused by: java.net.ConnectException: Connection refused (Connection
refused)
at java.net.PlainSocketImpl.socketConnect(Native Method)
at
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
at
sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:173)
at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
at
sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:264)
at sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:367)
at
sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1138)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1032)
at
sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
at
sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1546)
at
sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
at
sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
at
org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:153)
... 17 more
*
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Unable to connect ignite pods in Kubernetes using Ip-finder
Posted by Denis Magda <dm...@apache.org>.
Hi,
By some reason the pods cannot resolve an IP address of Kubernetes master that responses on service call. That’s the default name ‘kubernetes.default.svc.cluster.local’ and your DNS knows nothing about it.
If you cannot overcome the issue at the network configuration layer set an address of the master explicitly using this API method:
https://github.com/apache/ignite/blob/master/modules/kubernetes/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/kubernetes/TcpDiscoveryKubernetesIpFinder.java#L215
—
Denis
> On Nov 7, 2017, at 4:31 AM, rishi007bansod <ri...@gmail.com> wrote:
>
> Hi,
> I have initialized my pods as per steps in link
> https://apacheignite.readme.io/docs/kubernetes-deployment
> <https://apacheignite.readme.io/docs/kubernetes-deployment> . I have also
> tried setting Kubernetes network to *1. flannel 2. weave net*. But I get
> same error in both cases. Following are error logs :
>
> *Flannel Error
> *
>
> [11:39:30,256][SEVERE][main][TcpDiscoverySpi] Failed to get registered
> addresses from IP finder on start (retrying every 2000 ms).
> class org.apache.ignite.spi.IgniteSpiException: Failed to retrieve Ignite
> pods IP addresses.
> at
> org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:172)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.registeredAddresses(TcpDiscoverySpi.java:1613)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.resolvedAddresses(TcpDiscoverySpi.java:1562)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.sendJoinRequestMessage(ServerImpl.java:974)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.joinTopology(ServerImpl.java:837)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.spiStart(ServerImpl.java:351)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:1850)
> at
> org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:268)
> at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:685)
> at
> org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1626)
> at
> org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:924)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1799)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1602)
> at
> org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1042)
> at
> org.apache.ignite.internal.IgnitionEx.startConfigurations(IgnitionEx.java:964)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:850)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:749)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:619)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:589)
> at org.apache.ignite.Ignition.start(Ignition.java:347)
> at
> org.apache.ignite.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:302)
> Caused by: java.net.UnknownHostException:
> kubernetes.default.svc.cluster.local
> at
> java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:184)
> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
> at java.net.Socket.connect(Socket.java:589)
> at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:668)
> at
> sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:173)
> at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
> at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
> at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
> at
> sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:264)
> at sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:367)
> at
> sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
> at
> sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1138)
> at
> sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1032)
> at
> sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
> at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1546)
> at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
> at
> sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
> at
> org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:153)
> ... 20 more
>
>
>
>
> *Weave Net Error
> *
>
> [12:14:59] Security status [authentication=off, tls/ssl=off]
> [12:15:05,243][SEVERE][main][TcpDiscoverySpi] Failed to get registered
> addresses from IP finder on start (retrying every 2000 ms).
> class org.apache.ignite.spi.IgniteSpiException: Failed to retrieve Ignite
> pods IP addresses.
> at
> org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:172)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.registeredAddresses(TcpDiscoverySpi.java:1613)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.resolvedAddresses(TcpDiscoverySpi.java:1562)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.sendJoinRequestMessage(ServerImpl.java:974)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.joinTopology(ServerImpl.java:837)
> at
> org.apache.ignite.spi.discovery.tcp.ServerImpl.spiStart(ServerImpl.java:351)
> at
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:1850)
> at
> org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:268)
> at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:685)
> at
> org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1626)
> at
> org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:924)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1799)
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1602)
> at
> org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1042)
> at
> org.apache.ignite.internal.IgnitionEx.startConfigurations(IgnitionEx.java:964)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:850)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:749)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:619)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:589)
> at org.apache.ignite.Ignition.start(Ignition.java:347)
> at
> org.apache.ignite.startup.cmdline.CommandLineStartup.main(CommandLineStartup.java:302)
> Caused by: java.io.IOException: Server returned HTTP response code: 403 for
> URL:
> https://kubernetes.default.svc.cluster.local:443/api/v1/namespaces/default/endpoints/ignite
> at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1876)
> at
> sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
> at
> sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
> at
> org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder.getRegisteredAddresses(TcpDiscoveryKubernetesIpFinder.java:153)
> ... 20 more
>
>
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/