You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@falcon.apache.org by Vishal Gupta <gv...@gmail.com> on 2016/12/22 05:17:53 UTC
Falcon issue after Kerberos implementation
Hello Team,
We are implementing Kerberos at one the clusters.
After Kerberos implementation looks like Falcon while using SPNEGO to
authenticate the user, is failing while authenticating and connecting to
Oozie.
Any suggestions please. Details below..
2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//entities/delete/process/edmhdpif-oozie-icbdev-th]
~ Unable to reach workflow engine for deletion or deletion failed
(AbstractEntityManager:265)
org.apache.falcon.FalconException: IO_ERROR : java.io.IOException: Error
while connecting Oozie server. No of retries = 1. Exception = null
at
org.apache.falcon.workflow.engine.OozieWorkflowEngine.findBundles(OozieWorkflowEngine.java:303)
at
org.apache.falcon.workflow.engine.OozieWorkflowEngine.doBundleAction(OozieWorkflowEngine.java:377)
at
org.apache.falcon.workflow.engine.OozieWorkflowEngine.doBundleAction(OozieWorkflowEngine.java:371)
at
org.apache.falcon.workflow.engine.OozieWorkflowEngine.delete(OozieWorkflowEngine.java:355)
at
org.apache.falcon.resource.AbstractEntityManager.delete(AbstractEntityManager.java:252)
at
org.apache.falcon.resource.ConfigSyncService.delete(ConfigSyncService.java:62)
at sun.reflect.GeneratedMethodAccessor88.invoke(Unknown
Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.falcon.resource.channel.IPCChannel.invoke(IPCChannel.java:49)
at
org.apache.falcon.resource.proxy.SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityManagerProxy.java:230)
at
org.apache.falcon.resource.proxy.SchedulableEntityManagerProxy$EntityProxy.execute(SchedulableEntityManagerProxy.java:575)
at
org.apache.falcon.resource.proxy.SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.java:219)
at
org.apache.falcon.resource.proxy.SchedulableEntityManagerProxy.delete_aroundBody2(SchedulableEntityManagerProxy.java:232)
at
org.apache.falcon.resource.proxy.SchedulableEntityManagerProxy$AjcClosure3.run(SchedulableEntityManagerProxy.java:1)
at
org.aspectj.runtime.reflect.JoinPointImpl.proceed(JoinPointImpl.java:149)
at
org.apache.falcon.aspect.AbstractFalconAspect.logAroundMonitored(AbstractFalconAspect.java:51)
at
org.apache.falcon.resource.proxy.SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.java:206)
at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown
Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
at
com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185)
at
com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)
at
com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288)
at
com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
at
com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)
at
com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
at
com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
at
com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
at
com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
at
com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
at
com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
at
com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)
at
com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537)
at
com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699)
at
javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
at
org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
at
org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
at
org.apache.falcon.security.FalconAuthorizationFilter.doFilter(FalconAuthorizationFilter.java:106)
at
org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at
org.apache.falcon.security.FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:184)
at
org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
at
org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
at
org.apache.falcon.security.FalconAuthenticationFilter.doFilter(FalconAuthenticationFilter.java:193)
at
org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at
org.apache.falcon.security.FalconAuditFilter.doFilter(FalconAuditFilter.java:64)
at
org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767)
at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at
org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
server. No of retries = 1. Exception = null
at
org.apache.oozie.client.OozieClient.validateWSVersion(OozieClient.java:374)
at
org.apache.oozie.client.OozieClient.createURL(OozieClient.java:459)
at
org.apache.oozie.client.OozieClient.access$000(OozieClient.java:80)
at
org.apache.oozie.client.OozieClient$ClientCallable.call(OozieClient.java:555)
at
org.apache.oozie.client.OozieClient.getBundleJobsInfo(OozieClient.java:2149)
at
org.apache.oozie.client.ProxyOozieClient.access$2901(ProxyOozieClient.java:48)
at
org.apache.oozie.client.ProxyOozieClient$29.call(ProxyOozieClient.java:598)
at
org.apache.oozie.client.ProxyOozieClient$29.call(ProxyOozieClient.java:596)
at
org.apache.oozie.client.OozieClient.doAs(OozieClient.java:244)
at
org.apache.oozie.client.ProxyOozieClient.getBundleJobsInfo(ProxyOozieClient.java:596)
at
org.apache.falcon.workflow.engine.OozieWorkflowEngine.findBundles(OozieWorkflowEngine.java:290)
... 61 more
Caused by: java.io.IOException: Error while connecting Oozie server. No of
retries = 1. Exception = null
at
org.apache.oozie.client.retry.ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
at
org.apache.oozie.client.OozieClient.createRetryableConnection(OozieClient.java:504)
at
org.apache.oozie.client.OozieClient.getSupportedProtocolVersions(OozieClient.java:384)
at
org.apache.oozie.client.OozieClient.validateWSVersion(OozieClient.java:344)
... 71 more
Caused by: java.lang.reflect.UndeclaredThrowableException
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1672)
at
org.apache.oozie.client.ProxyOozieClient.createConnection(ProxyOozieClient.java:79)
at
org.apache.oozie.client.OozieClient$1.doExecute(OozieClient.java:507)
at
org.apache.oozie.client.retry.ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
... 74 more
Caused by: AUTHENTICATION : Could not authenticate, GSSException: No valid
credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at
org.apache.oozie.client.AuthOozieClient.createConnection(AuthOozieClient.java:136)
at
org.apache.oozie.client.ProxyOozieClient.access$001(ProxyOozieClient.java:48)
at
org.apache.oozie.client.ProxyOozieClient$1.run(ProxyOozieClient.java:81)
at
org.apache.oozie.client.ProxyOozieClient$1.run(ProxyOozieClient.java:79)
at java.security.AccessController.doPrivileged(Native
Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
... 77 more
Caused by:
org.apache.hadoop.security.authentication.client.AuthenticationException:
GSSException: No valid credentials provided (Mechanism level: Failed to
find any Kerberos tgt)
at
org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
at
org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
at
org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
at
org.apache.oozie.client.AuthOozieClient.createConnection(AuthOozieClient.java:127)
... 83 more
Caused by: GSSException: No valid credentials provided (Mechanism level:
Failed to find any Kerberos tgt)
at
sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147)
at
sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
at
sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187)
at
sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
at
sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
at
org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
at
org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
at java.security.AccessController.doPrivileged(Native
Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
... 86 more
Re: Falcon issue after Kerberos implementation
Posted by Vishal Gupta <gv...@gmail.com>.
Hello Venkat,
HDP 2.3.3 release notes says that patch has been applied for FALCON-954
(which is the exact issue as we have). So looks like it is something else.
Regards,
Vishal.
On Thu, Dec 22, 2016 at 2:11 PM, Vishal Gupta <gv...@gmail.com> wrote:
> sorry typo, we are on 0.6.*1*
>
> Regards,
> Vishal.
>
> On Thu, Dec 22, 2016 at 2:09 PM, Vishal Gupta <gv...@gmail.com> wrote:
>
>> Thanks a lot Venkat for quick help.
>>
>> I will look into it. We will contact Hortonworks for patch or check the
>> possibility of upgrade. we are on 0.6.0 version as of now.
>>
>> Regards
>> Vishal.
>>
>>
>> On Thu, Dec 22, 2016 at 1:58 PM, Venkat Ranganathan <
>> vranganathan@hortonworks.com> wrote:
>>
>>> This is addressed as part of FALCON-2025. It is fixed in 0.10 release
>>>
>>>
>>>
>>> Thanks
>>>
>>> Venkat
>>>
>>>
>>>
>>> *From: *Vishal Gupta <gv...@gmail.com>
>>> *Date: *Wednesday, December 21, 2016 at 9:37 PM
>>> *To: *"dev@falcon.apache.org" <de...@falcon.apache.org>, Venkat
>>> Ranganathan <vr...@hortonworks.com>
>>>
>>> *Subject: *Re: Falcon issue after Kerberos implementation
>>>
>>>
>>>
>>> Hello Venkat,
>>>
>>>
>>>
>>> Thanks for quick response. Could you please guide to any contacts in HWX
>>> or any online server where we can find the patch / answer.
>>>
>>>
>>>
>>> Do you know in which version this issue is being resolved.
>>>
>>>
>>>
>>> Thanks in advance.
>>>
>>>
>>>
>>> Regards,
>>>
>>> Vishal.
>>>
>>>
>>>
>>>
>>>
>>> On Thu, Dec 22, 2016 at 1:34 PM, Venkat Ranganathan <
>>> vranganathan@hortonworks.com> wrote:
>>>
>>> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
>>> valid credentials provided (Mechanism level: Failed to find any Kerberos
>>> tgt)
>>>
>>> This shows that the Falcon kerberos context become invalidated. There
>>> was a fix in later versions and patch should be available from HWX support
>>> for the older versions
>>>
>>> Thanks
>>> Venkat
>>> ________________________________________
>>> From: Vishal Gupta <gv...@gmail.com>
>>> Sent: Wednesday, December 21, 2016 9:19 PM
>>> To: dev@falcon.apache.org
>>> Subject: Re: Falcon issue after Kerberos implementation
>>>
>>>
>>> Cluster is HDP 2.3.2
>>>
>>> On Thu, Dec 22, 2016 at 1:17 PM, Vishal Gupta <gv...@gmail.com>
>>> wrote:
>>>
>>> > Hello Team,
>>> > We are implementing Kerberos at one the clusters.
>>> >
>>> > After Kerberos implementation looks like Falcon while using SPNEGO to
>>> > authenticate the user, is failing while authenticating and connecting
>>> to
>>> > Oozie.
>>> >
>>> >
>>> >
>>> > Any suggestions please. Details below..
>>> >
>>> >
>>> > 2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
>>> > 737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//
>>> > entities/delete/process/edmhdpif-oozie-icbdev-th] ~ Unable to reach
>>> > workflow engine for deletion or deletion failed
>>> (AbstractEntityManager:265)
>>> >
>>> > org.apache.falcon.FalconException: IO_ERROR : java.io.IOException:
>>> Error
>>> > while connecting Oozie server. No of retries = 1. Exception = null
>>> >
>>> > at org.apache.falcon.workflow.eng
>>> ine.OozieWorkflowEngine.
>>> > findBundles(OozieWorkflowEngine.java:303)
>>> >
>>> > at org.apache.falcon.workflow.eng
>>> ine.OozieWorkflowEngine.
>>> > doBundleAction(OozieWorkflowEngine.java:377)
>>> >
>>> > at org.apache.falcon.workflow.eng
>>> ine.OozieWorkflowEngine.
>>> > doBundleAction(OozieWorkflowEngine.java:371)
>>> >
>>> > at org.apache.falcon.workflow.eng
>>> ine.OozieWorkflowEngine.
>>> > delete(OozieWorkflowEngine.java:355)
>>> >
>>> > at org.apache.falcon.resource.
>>> > AbstractEntityManager.delete(AbstractEntityManager.java:252)
>>> >
>>> > at org.apache.falcon.resource.Con
>>> figSyncService.delete(
>>> > ConfigSyncService.java:62)
>>> >
>>> > at sun.reflect.GeneratedMethodAcc
>>> essor88.invoke(Unknown
>>> > Source)
>>> >
>>> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
>>> > DelegatingMethodAccessorImpl.java:43)
>>> >
>>> > at java.lang.reflect.Method.invoke(Method.java:498)
>>> >
>>> > at org.apache.falcon.resource.cha
>>> nnel.IPCChannel.invoke(
>>> > IPCChannel.java:49)
>>> >
>>> > at org.apache.falcon.resource.proxy.
>>> > SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityM
>>> anagerProxy.
>>> > java:230)
>>> >
>>> > at org.apache.falcon.resource.proxy.
>>> > SchedulableEntityManagerProxy$EntityProxy.execute(
>>> > SchedulableEntityManagerProxy.java:575)
>>> >
>>> > at org.apache.falcon.resource.proxy.
>>> > SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.
>>> > java:219)
>>> >
>>> > at org.apache.falcon.resource.proxy.
>>> > SchedulableEntityManagerProxy.delete_aroundBody2(
>>> > SchedulableEntityManagerProxy.java:232)
>>> >
>>> > at org.apache.falcon.resource.proxy.
>>> > SchedulableEntityManagerProxy$AjcClosure3.run(
>>> > SchedulableEntityManagerProxy.java:1)
>>> >
>>> > at org.aspectj.runtime.reflect.JoinPointImpl.proceed(
>>> > JoinPointImpl.java:149)
>>> >
>>> > at org.apache.falcon.aspect.AbstractFalconAspect.
>>> > logAroundMonitored(AbstractFalconAspect.java:51)
>>> >
>>> > at org.apache.falcon.resource.proxy.
>>> > SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.
>>> > java:206)
>>> >
>>> > at sun.reflect.GeneratedMethodAcc
>>> essor87.invoke(Unknown
>>> > Source)
>>> >
>>> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
>>> > DelegatingMethodAccessorImpl.java:43)
>>> >
>>> > at java.lang.reflect.Method.invoke(Method.java:498)
>>> >
>>>
>>> > at com.sun.jersey.spi.container.
>>> > JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
>>> >
>>> > at com.sun.jersey.server.impl.model.method.dispatch.
>>> > AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(
>>> > AbstractResourceMethodDispatchProvider.java:185)
>>> >
>>> > at com.sun.jersey.server.impl.model.method.dispatch.
>>> > ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.
>>> > java:75)
>>> >
>>> > at com.sun.jersey.server.impl.uri
>>> .rules.HttpMethodRule.
>>> > accept(HttpMethodRule.java:288)
>>> >
>>> > at com.sun.jersey.server.impl.uri
>>> .rules.RightHandPathRule.
>>> > accept(RightHandPathRule.java:147)
>>> >
>>> > at com.sun.jersey.server.impl.uri
>>> .rules.ResourceClassRule.
>>> > accept(ResourceClassRule.java:108)
>>> >
>>> > at com.sun.jersey.server.impl.uri
>>> .rules.RightHandPathRule.
>>> > accept(RightHandPathRule.java:147)
>>> >
>>> > at com.sun.jersey.server.impl.uri.rules.
>>> > RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
>>> >
>>> > at com.sun.jersey.server.impl.application.
>>> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
>>> >
>>> > at com.sun.jersey.server.impl.application.
>>> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
>>> >
>>> > at com.sun.jersey.server.impl.application.
>>> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
>>> >
>>> > at com.sun.jersey.server.impl.application.
>>> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
>>> >
>>> > at com.sun.jersey.spi.container.
>>> > servlet.WebComponent.service(WebComponent.java:416)
>>> >
>>> > at com.sun.jersey.spi.container.s
>>> ervlet.ServletContainer.
>>> > service(ServletContainer.java:537)
>>> >
>>> > at com.sun.jersey.spi.container.s
>>> ervlet.ServletContainer.
>>> > service(ServletContainer.java:699)
>>> >
>>> > at javax.servlet.http.HttpServlet.service(
>>> > HttpServlet.java:820)
>>> >
>>> > at org.mortbay.jetty.servlet.ServletHolder.handle(
>>> > ServletHolder.java:511)
>>> >
>>> > at org.mortbay.jetty.servlet.Serv
>>> letHandler$CachedChain.
>>> > doFilter(ServletHandler.java:1221)
>>> >
>>> > at org.apache.falcon.security.Fal
>>> conAuthorizationFilter.
>>> > doFilter(FalconAuthorizationFilter.java:106)
>>> >
>>> > at org.mortbay.jetty.servlet.Serv
>>> letHandler$CachedChain.
>>> > doFilter(ServletHandler.java:1212)
>>> >
>>> > at org.apache.falcon.security.
>>> > FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFi
>>> lter.java:184)
>>> >
>>> > at org.apache.hadoop.security.authentication.server.
>>> > AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
>>> >
>>> > at org.apache.hadoop.security.authentication.server.
>>> > AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
>>> >
>>> > at org.apache.falcon.security.Fal
>>> conAuthenticationFilter.
>>> > doFilter(FalconAuthenticationFilter.java:193)
>>> >
>>> > at org.mortbay.jetty.servlet.Serv
>>> letHandler$CachedChain.
>>> > doFilter(ServletHandler.java:1212)
>>> >
>>> > at org.apache.falcon.security.Fal
>>> conAuditFilter.doFilter(
>>> > FalconAuditFilter.java:64)
>>> >
>>> > at org.mortbay.jetty.servlet.Serv
>>> letHandler$CachedChain.
>>> > doFilter(ServletHandler.java:1212)
>>> >
>>> > at org.mortbay.jetty.servlet.ServletHandler.handle(
>>> > ServletHandler.java:399)
>>> >
>>> > at org.mortbay.jetty.security.SecurityHandler.handle(
>>> > SecurityHandler.java:216)
>>> >
>>> > at org.mortbay.jetty.servlet.SessionHandler.handle(
>>> > SessionHandler.java:182)
>>> >
>>> > at org.mortbay.jetty.handler.ContextHandler.handle(
>>> > ContextHandler.java:767)
>>> >
>>> > at org.mortbay.jetty.webapp.WebAppContext.handle(
>>> > WebAppContext.java:450)
>>> >
>>> > at org.mortbay.jetty.handler.HandlerWrapper.handle(
>>>
>>> > HandlerWrapper.java:152)
>>> >
>>> > at org.mortbay.jetty.Server.handle(Server.java:326)
>>> >
>>> > at org.mortbay.jetty.HttpConnection.handleRequest(
>>> > HttpConnection.java:542)
>>> >
>>> > at org.mortbay.jetty.HttpConnection$RequestHandler.
>>> > headerComplete(HttpConnection.java:928)
>>> >
>>> > at org.mortbay.jetty.HttpParser.
>>> > parseNext(HttpParser.java:549)
>>> >
>>> > at org.mortbay.jetty.HttpParser.p
>>> arseAvailable(HttpParser.
>>> > java:212)
>>> >
>>> > at org.mortbay.jetty.HttpConnection.handle(
>>> > HttpConnection.java:404)
>>> >
>>> > at org.mortbay.jetty.bio.SocketConnector$Connection.
>>> > run(SocketConnector.java:228)
>>> >
>>> > at org.mortbay.thread.QueuedThreadPool$PoolThread.
>>> > run(QueuedThreadPool.java:582)
>>> >
>>> > Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
>>> > server. No of retries = 1. Exception = null
>>> >
>>> > at org.apache.oozie.client.OozieC
>>> lient.validateWSVersion(
>>> > OozieClient.java:374)
>>> >
>>> > at org.apache.oozie.client.OozieClient.createURL(
>>> > OozieClient.java:459)
>>> >
>>> > at org.apache.oozie.client.OozieClient.access$000(
>>> > OozieClient.java:80)
>>> >
>>> > at org.apache.oozie.client.OozieClient$ClientCallable.
>>> > call(OozieClient.java:555)
>>> >
>>> > at org.apache.oozie.client.OozieC
>>> lient.getBundleJobsInfo(
>>> > OozieClient.java:2149)
>>> >
>>> > at org.apache.oozie.client.ProxyO
>>> ozieClient.access$2901(
>>> > ProxyOozieClient.java:48)
>>> >
>>> > at org.apache.oozie.client.ProxyOozieClient$29.call(
>>> > ProxyOozieClient.java:598)
>>> >
>>> > at org.apache.oozie.client.ProxyOozieClient$29.call(
>>> > ProxyOozieClient.java:596)
>>> >
>>> > at org.apache.oozie.client.OozieC
>>> lient.doAs(OozieClient.
>>> > java:244)
>>> >
>>> > at org.apache.oozie.client.ProxyOozieClient.
>>> > getBundleJobsInfo(ProxyOozieClient.java:596)
>>> >
>>> > at org.apache.falcon.workflow.eng
>>> ine.OozieWorkflowEngine.
>>> > findBundles(OozieWorkflowEngine.java:290)
>>> >
>>> > ... 61 more
>>> >
>>> > Caused by: java.io.IOException: Error while connecting Oozie server.
>>> No of
>>> > retries = 1. Exception = null
>>> >
>>> > at org.apache.oozie.client.retry.
>>> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
>>> >
>>> > at org.apache.oozie.client.OozieClient.
>>> > createRetryableConnection(OozieClient.java:504)
>>> >
>>> > at org.apache.oozie.client.OozieClient.
>>> > getSupportedProtocolVersions(OozieClient.java:384)
>>> >
>>> > at org.apache.oozie.client.OozieC
>>> lient.validateWSVersion(
>>> > OozieClient.java:344)
>>> >
>>> > ... 71 more
>>> >
>>> > Caused by: java.lang.reflect.UndeclaredThrowableException
>>> >
>>> > at org.apache.hadoop.security.Use
>>> rGroupInformation.doAs(
>>> > UserGroupInformation.java:1672)
>>> >
>>> > at org.apache.oozie.client.ProxyOozieClient.
>>> > createConnection(ProxyOozieClient.java:79)
>>> >
>>> > at org.apache.oozie.client.OozieClient$1.doExecute(
>>> > OozieClient.java:507)
>>> >
>>> > at org.apache.oozie.client.retry.
>>> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
>>> >
>>> > ... 74 more
>>> >
>>> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
>>> valid
>>> > credentials provided (Mechanism level: Failed to find any Kerberos tgt)
>>> >
>>> > at org.apache.oozie.client.AuthOozieClient.
>>> > createConnection(AuthOozieClient.java:136)
>>> >
>>> > at org.apache.oozie.client.ProxyO
>>> ozieClient.access$001(
>>> > ProxyOozieClient.java:48)
>>> >
>>> > at org.apache.oozie.client.ProxyOozieClient$1.run(
>>> > ProxyOozieClient.java:81)
>>> >
>>> > at org.apache.oozie.client.ProxyOozieClient$1.run(
>>> > ProxyOozieClient.java:79)
>>> >
>>> > at java.security.AccessController.doPrivileged(Native
>>> > Method)
>>> >
>>> > at javax.security.auth.Subject.doAs(Subject.java:422)
>>> >
>>> > at org.apache.hadoop.security.Use
>>> rGroupInformation.doAs(
>>> > UserGroupInformation.java:1657)
>>> >
>>> > ... 77 more
>>> >
>>> > Caused by: org.apache.hadoop.security.aut
>>> hentication.client.AuthenticationException:
>>> > GSSException: No valid credentials provided (Mechanism level: Failed to
>>> > find any Kerberos tgt)
>>> >
>>> > at org.apache.hadoop.security.authentication.client.
>>> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
>>> >
>>> > at org.apache.hadoop.security.authentication.client.
>>> > KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
>>> >
>>> > at org.apache.hadoop.security.authentication.client.
>>> > AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
>>> >
>>> > at org.apache.oozie.client.AuthOozieClient.
>>> > createConnection(AuthOozieClient.java:127)
>>> >
>>> > ... 83 more
>>> >
>>> > Caused by: GSSException: No valid credentials provided (Mechanism
>>> level:
>>> > Failed to find any Kerberos tgt)
>>> >
>>> > at sun.security.jgss.krb5.Krb5Ini
>>> tCredential.getInstance(
>>> > Krb5InitCredential.java:147)
>>> >
>>> > at sun.security.jgss.krb5.Krb5MechFactory.
>>> > getCredentialElement(Krb5MechFactory.java:122)
>>> >
>>> > at sun.security.jgss.krb5.Krb5MechFactory.
>>> > getMechanismContext(Krb5MechFactory.java:187)
>>> >
>>> > at sun.security.jgss.GSSManagerIm
>>> pl.getMechanismContext(
>>> > GSSManagerImpl.java:224)
>>> >
>>> > at sun.security.jgss.GSSContextImpl.initSecContext(
>>> > GSSContextImpl.java:212)
>>> >
>>> > at sun.security.jgss.GSSContextImpl.initSecContext(
>>> > GSSContextImpl.java:179)
>>> >
>>> > at org.apache.hadoop.security.authentication.client.
>>> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
>>> >
>>> > at org.apache.hadoop.security.authentication.client.
>>> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
>>> >
>>> > at java.security.AccessController.doPrivileged(Native
>>> > Method)
>>> >
>>> > at javax.security.auth.Subject.doAs(Subject.java:422)
>>> >
>>> > at org.apache.hadoop.security.authentication.client.
>>> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
>>> >
>>> > ... 86 more
>>> >
>>> >
>>> >
>>> >
>>> >
>>>
>>>
>>>
>>
>>
>
Re: Falcon issue after Kerberos implementation
Posted by Vishal Gupta <gv...@gmail.com>.
sorry typo, we are on 0.6.*1*
Regards,
Vishal.
On Thu, Dec 22, 2016 at 2:09 PM, Vishal Gupta <gv...@gmail.com> wrote:
> Thanks a lot Venkat for quick help.
>
> I will look into it. We will contact Hortonworks for patch or check the
> possibility of upgrade. we are on 0.6.0 version as of now.
>
> Regards
> Vishal.
>
>
> On Thu, Dec 22, 2016 at 1:58 PM, Venkat Ranganathan <
> vranganathan@hortonworks.com> wrote:
>
>> This is addressed as part of FALCON-2025. It is fixed in 0.10 release
>>
>>
>>
>> Thanks
>>
>> Venkat
>>
>>
>>
>> *From: *Vishal Gupta <gv...@gmail.com>
>> *Date: *Wednesday, December 21, 2016 at 9:37 PM
>> *To: *"dev@falcon.apache.org" <de...@falcon.apache.org>, Venkat
>> Ranganathan <vr...@hortonworks.com>
>>
>> *Subject: *Re: Falcon issue after Kerberos implementation
>>
>>
>>
>> Hello Venkat,
>>
>>
>>
>> Thanks for quick response. Could you please guide to any contacts in HWX
>> or any online server where we can find the patch / answer.
>>
>>
>>
>> Do you know in which version this issue is being resolved.
>>
>>
>>
>> Thanks in advance.
>>
>>
>>
>> Regards,
>>
>> Vishal.
>>
>>
>>
>>
>>
>> On Thu, Dec 22, 2016 at 1:34 PM, Venkat Ranganathan <
>> vranganathan@hortonworks.com> wrote:
>>
>> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
>> valid credentials provided (Mechanism level: Failed to find any Kerberos
>> tgt)
>>
>> This shows that the Falcon kerberos context become invalidated. There
>> was a fix in later versions and patch should be available from HWX support
>> for the older versions
>>
>> Thanks
>> Venkat
>> ________________________________________
>> From: Vishal Gupta <gv...@gmail.com>
>> Sent: Wednesday, December 21, 2016 9:19 PM
>> To: dev@falcon.apache.org
>> Subject: Re: Falcon issue after Kerberos implementation
>>
>>
>> Cluster is HDP 2.3.2
>>
>> On Thu, Dec 22, 2016 at 1:17 PM, Vishal Gupta <gv...@gmail.com>
>> wrote:
>>
>> > Hello Team,
>> > We are implementing Kerberos at one the clusters.
>> >
>> > After Kerberos implementation looks like Falcon while using SPNEGO to
>> > authenticate the user, is failing while authenticating and connecting to
>> > Oozie.
>> >
>> >
>> >
>> > Any suggestions please. Details below..
>> >
>> >
>> > 2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
>> > 737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//
>> > entities/delete/process/edmhdpif-oozie-icbdev-th] ~ Unable to reach
>> > workflow engine for deletion or deletion failed
>> (AbstractEntityManager:265)
>> >
>> > org.apache.falcon.FalconException: IO_ERROR : java.io.IOException:
>> Error
>> > while connecting Oozie server. No of retries = 1. Exception = null
>> >
>> > at org.apache.falcon.workflow.eng
>> ine.OozieWorkflowEngine.
>> > findBundles(OozieWorkflowEngine.java:303)
>> >
>> > at org.apache.falcon.workflow.eng
>> ine.OozieWorkflowEngine.
>> > doBundleAction(OozieWorkflowEngine.java:377)
>> >
>> > at org.apache.falcon.workflow.eng
>> ine.OozieWorkflowEngine.
>> > doBundleAction(OozieWorkflowEngine.java:371)
>> >
>> > at org.apache.falcon.workflow.eng
>> ine.OozieWorkflowEngine.
>> > delete(OozieWorkflowEngine.java:355)
>> >
>> > at org.apache.falcon.resource.
>> > AbstractEntityManager.delete(AbstractEntityManager.java:252)
>> >
>> > at org.apache.falcon.resource.ConfigSyncService.delete(
>> > ConfigSyncService.java:62)
>> >
>> > at sun.reflect.GeneratedMethodAccessor88.invoke(Unknown
>> > Source)
>> >
>> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
>> > DelegatingMethodAccessorImpl.java:43)
>> >
>> > at java.lang.reflect.Method.invoke(Method.java:498)
>> >
>> > at org.apache.falcon.resource.cha
>> nnel.IPCChannel.invoke(
>> > IPCChannel.java:49)
>> >
>> > at org.apache.falcon.resource.proxy.
>> > SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityM
>> anagerProxy.
>> > java:230)
>> >
>> > at org.apache.falcon.resource.proxy.
>> > SchedulableEntityManagerProxy$EntityProxy.execute(
>> > SchedulableEntityManagerProxy.java:575)
>> >
>> > at org.apache.falcon.resource.proxy.
>> > SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.
>> > java:219)
>> >
>> > at org.apache.falcon.resource.proxy.
>> > SchedulableEntityManagerProxy.delete_aroundBody2(
>> > SchedulableEntityManagerProxy.java:232)
>> >
>> > at org.apache.falcon.resource.proxy.
>> > SchedulableEntityManagerProxy$AjcClosure3.run(
>> > SchedulableEntityManagerProxy.java:1)
>> >
>> > at org.aspectj.runtime.reflect.JoinPointImpl.proceed(
>> > JoinPointImpl.java:149)
>> >
>> > at org.apache.falcon.aspect.AbstractFalconAspect.
>> > logAroundMonitored(AbstractFalconAspect.java:51)
>> >
>> > at org.apache.falcon.resource.proxy.
>> > SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.
>> > java:206)
>> >
>> > at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown
>> > Source)
>> >
>> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
>> > DelegatingMethodAccessorImpl.java:43)
>> >
>> > at java.lang.reflect.Method.invoke(Method.java:498)
>> >
>>
>> > at com.sun.jersey.spi.container.
>> > JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
>> >
>> > at com.sun.jersey.server.impl.model.method.dispatch.
>> > AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(
>> > AbstractResourceMethodDispatchProvider.java:185)
>> >
>> > at com.sun.jersey.server.impl.model.method.dispatch.
>> > ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.
>> > java:75)
>> >
>> > at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.
>> > accept(HttpMethodRule.java:288)
>> >
>> > at com.sun.jersey.server.impl.uri
>> .rules.RightHandPathRule.
>> > accept(RightHandPathRule.java:147)
>> >
>> > at com.sun.jersey.server.impl.uri
>> .rules.ResourceClassRule.
>> > accept(ResourceClassRule.java:108)
>> >
>> > at com.sun.jersey.server.impl.uri
>> .rules.RightHandPathRule.
>> > accept(RightHandPathRule.java:147)
>> >
>> > at com.sun.jersey.server.impl.uri.rules.
>> > RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
>> >
>> > at com.sun.jersey.server.impl.application.
>> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
>> >
>> > at com.sun.jersey.server.impl.application.
>> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
>> >
>> > at com.sun.jersey.server.impl.application.
>> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
>> >
>> > at com.sun.jersey.server.impl.application.
>> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
>> >
>> > at com.sun.jersey.spi.container.
>> > servlet.WebComponent.service(WebComponent.java:416)
>> >
>> > at com.sun.jersey.spi.container.s
>> ervlet.ServletContainer.
>> > service(ServletContainer.java:537)
>> >
>> > at com.sun.jersey.spi.container.s
>> ervlet.ServletContainer.
>> > service(ServletContainer.java:699)
>> >
>> > at javax.servlet.http.HttpServlet.service(
>> > HttpServlet.java:820)
>> >
>> > at org.mortbay.jetty.servlet.ServletHolder.handle(
>> > ServletHolder.java:511)
>> >
>> > at org.mortbay.jetty.servlet.Serv
>> letHandler$CachedChain.
>> > doFilter(ServletHandler.java:1221)
>> >
>> > at org.apache.falcon.security.Fal
>> conAuthorizationFilter.
>> > doFilter(FalconAuthorizationFilter.java:106)
>> >
>> > at org.mortbay.jetty.servlet.Serv
>> letHandler$CachedChain.
>> > doFilter(ServletHandler.java:1212)
>> >
>> > at org.apache.falcon.security.
>> > FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFi
>> lter.java:184)
>> >
>> > at org.apache.hadoop.security.authentication.server.
>> > AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
>> >
>> > at org.apache.hadoop.security.authentication.server.
>> > AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
>> >
>> > at org.apache.falcon.security.Fal
>> conAuthenticationFilter.
>> > doFilter(FalconAuthenticationFilter.java:193)
>> >
>> > at org.mortbay.jetty.servlet.Serv
>> letHandler$CachedChain.
>> > doFilter(ServletHandler.java:1212)
>> >
>> > at org.apache.falcon.security.Fal
>> conAuditFilter.doFilter(
>> > FalconAuditFilter.java:64)
>> >
>> > at org.mortbay.jetty.servlet.Serv
>> letHandler$CachedChain.
>> > doFilter(ServletHandler.java:1212)
>> >
>> > at org.mortbay.jetty.servlet.ServletHandler.handle(
>> > ServletHandler.java:399)
>> >
>> > at org.mortbay.jetty.security.SecurityHandler.handle(
>> > SecurityHandler.java:216)
>> >
>> > at org.mortbay.jetty.servlet.SessionHandler.handle(
>> > SessionHandler.java:182)
>> >
>> > at org.mortbay.jetty.handler.ContextHandler.handle(
>> > ContextHandler.java:767)
>> >
>> > at org.mortbay.jetty.webapp.WebAppContext.handle(
>> > WebAppContext.java:450)
>> >
>> > at org.mortbay.jetty.handler.HandlerWrapper.handle(
>>
>> > HandlerWrapper.java:152)
>> >
>> > at org.mortbay.jetty.Server.handle(Server.java:326)
>> >
>> > at org.mortbay.jetty.HttpConnection.handleRequest(
>> > HttpConnection.java:542)
>> >
>> > at org.mortbay.jetty.HttpConnection$RequestHandler.
>> > headerComplete(HttpConnection.java:928)
>> >
>> > at org.mortbay.jetty.HttpParser.
>> > parseNext(HttpParser.java:549)
>> >
>> > at org.mortbay.jetty.HttpParser.p
>> arseAvailable(HttpParser.
>> > java:212)
>> >
>> > at org.mortbay.jetty.HttpConnection.handle(
>> > HttpConnection.java:404)
>> >
>> > at org.mortbay.jetty.bio.SocketConnector$Connection.
>> > run(SocketConnector.java:228)
>> >
>> > at org.mortbay.thread.QueuedThreadPool$PoolThread.
>> > run(QueuedThreadPool.java:582)
>> >
>> > Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
>> > server. No of retries = 1. Exception = null
>> >
>> > at org.apache.oozie.client.OozieC
>> lient.validateWSVersion(
>> > OozieClient.java:374)
>> >
>> > at org.apache.oozie.client.OozieClient.createURL(
>> > OozieClient.java:459)
>> >
>> > at org.apache.oozie.client.OozieClient.access$000(
>> > OozieClient.java:80)
>> >
>> > at org.apache.oozie.client.OozieClient$ClientCallable.
>> > call(OozieClient.java:555)
>> >
>> > at org.apache.oozie.client.OozieC
>> lient.getBundleJobsInfo(
>> > OozieClient.java:2149)
>> >
>> > at org.apache.oozie.client.ProxyO
>> ozieClient.access$2901(
>> > ProxyOozieClient.java:48)
>> >
>> > at org.apache.oozie.client.ProxyOozieClient$29.call(
>> > ProxyOozieClient.java:598)
>> >
>> > at org.apache.oozie.client.ProxyOozieClient$29.call(
>> > ProxyOozieClient.java:596)
>> >
>> > at org.apache.oozie.client.OozieC
>> lient.doAs(OozieClient.
>> > java:244)
>> >
>> > at org.apache.oozie.client.ProxyOozieClient.
>> > getBundleJobsInfo(ProxyOozieClient.java:596)
>> >
>> > at org.apache.falcon.workflow.eng
>> ine.OozieWorkflowEngine.
>> > findBundles(OozieWorkflowEngine.java:290)
>> >
>> > ... 61 more
>> >
>> > Caused by: java.io.IOException: Error while connecting Oozie server. No
>> of
>> > retries = 1. Exception = null
>> >
>> > at org.apache.oozie.client.retry.
>> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
>> >
>> > at org.apache.oozie.client.OozieClient.
>> > createRetryableConnection(OozieClient.java:504)
>> >
>> > at org.apache.oozie.client.OozieClient.
>> > getSupportedProtocolVersions(OozieClient.java:384)
>> >
>> > at org.apache.oozie.client.OozieC
>> lient.validateWSVersion(
>> > OozieClient.java:344)
>> >
>> > ... 71 more
>> >
>> > Caused by: java.lang.reflect.UndeclaredThrowableException
>> >
>> > at org.apache.hadoop.security.Use
>> rGroupInformation.doAs(
>> > UserGroupInformation.java:1672)
>> >
>> > at org.apache.oozie.client.ProxyOozieClient.
>> > createConnection(ProxyOozieClient.java:79)
>> >
>> > at org.apache.oozie.client.OozieClient$1.doExecute(
>> > OozieClient.java:507)
>> >
>> > at org.apache.oozie.client.retry.
>> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
>> >
>> > ... 74 more
>> >
>> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
>> valid
>> > credentials provided (Mechanism level: Failed to find any Kerberos tgt)
>> >
>> > at org.apache.oozie.client.AuthOozieClient.
>> > createConnection(AuthOozieClient.java:136)
>> >
>> > at org.apache.oozie.client.ProxyOozieClient.access$001(
>> > ProxyOozieClient.java:48)
>> >
>> > at org.apache.oozie.client.ProxyOozieClient$1.run(
>> > ProxyOozieClient.java:81)
>> >
>> > at org.apache.oozie.client.ProxyOozieClient$1.run(
>> > ProxyOozieClient.java:79)
>> >
>> > at java.security.AccessController.doPrivileged(Native
>> > Method)
>> >
>> > at javax.security.auth.Subject.doAs(Subject.java:422)
>> >
>> > at org.apache.hadoop.security.Use
>> rGroupInformation.doAs(
>> > UserGroupInformation.java:1657)
>> >
>> > ... 77 more
>> >
>> > Caused by: org.apache.hadoop.security.authentication.client.Authenticat
>> ionException:
>> > GSSException: No valid credentials provided (Mechanism level: Failed to
>> > find any Kerberos tgt)
>> >
>> > at org.apache.hadoop.security.authentication.client.
>> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
>> >
>> > at org.apache.hadoop.security.authentication.client.
>> > KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
>> >
>> > at org.apache.hadoop.security.authentication.client.
>> > AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
>> >
>> > at org.apache.oozie.client.AuthOozieClient.
>> > createConnection(AuthOozieClient.java:127)
>> >
>> > ... 83 more
>> >
>> > Caused by: GSSException: No valid credentials provided (Mechanism level:
>> > Failed to find any Kerberos tgt)
>> >
>> > at sun.security.jgss.krb5.Krb5Ini
>> tCredential.getInstance(
>> > Krb5InitCredential.java:147)
>> >
>> > at sun.security.jgss.krb5.Krb5MechFactory.
>> > getCredentialElement(Krb5MechFactory.java:122)
>> >
>> > at sun.security.jgss.krb5.Krb5MechFactory.
>> > getMechanismContext(Krb5MechFactory.java:187)
>> >
>> > at sun.security.jgss.GSSManagerIm
>> pl.getMechanismContext(
>> > GSSManagerImpl.java:224)
>> >
>> > at sun.security.jgss.GSSContextImpl.initSecContext(
>> > GSSContextImpl.java:212)
>> >
>> > at sun.security.jgss.GSSContextImpl.initSecContext(
>> > GSSContextImpl.java:179)
>> >
>> > at org.apache.hadoop.security.authentication.client.
>> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
>> >
>> > at org.apache.hadoop.security.authentication.client.
>> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
>> >
>> > at java.security.AccessController.doPrivileged(Native
>> > Method)
>> >
>> > at javax.security.auth.Subject.doAs(Subject.java:422)
>> >
>> > at org.apache.hadoop.security.authentication.client.
>> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
>> >
>> > ... 86 more
>> >
>> >
>> >
>> >
>> >
>>
>>
>>
>
>
Re: Falcon issue after Kerberos implementation
Posted by Vishal Gupta <gv...@gmail.com>.
Thanks a lot Venkat for quick help.
I will look into it. We will contact Hortonworks for patch or check the
possibility of upgrade. we are on 0.6.0 version as of now.
Regards
Vishal.
On Thu, Dec 22, 2016 at 1:58 PM, Venkat Ranganathan <
vranganathan@hortonworks.com> wrote:
> This is addressed as part of FALCON-2025. It is fixed in 0.10 release
>
>
>
> Thanks
>
> Venkat
>
>
>
> *From: *Vishal Gupta <gv...@gmail.com>
> *Date: *Wednesday, December 21, 2016 at 9:37 PM
> *To: *"dev@falcon.apache.org" <de...@falcon.apache.org>, Venkat Ranganathan
> <vr...@hortonworks.com>
>
> *Subject: *Re: Falcon issue after Kerberos implementation
>
>
>
> Hello Venkat,
>
>
>
> Thanks for quick response. Could you please guide to any contacts in HWX
> or any online server where we can find the patch / answer.
>
>
>
> Do you know in which version this issue is being resolved.
>
>
>
> Thanks in advance.
>
>
>
> Regards,
>
> Vishal.
>
>
>
>
>
> On Thu, Dec 22, 2016 at 1:34 PM, Venkat Ranganathan <
> vranganathan@hortonworks.com> wrote:
>
> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
> valid credentials provided (Mechanism level: Failed to find any Kerberos
> tgt)
>
> This shows that the Falcon kerberos context become invalidated. There
> was a fix in later versions and patch should be available from HWX support
> for the older versions
>
> Thanks
> Venkat
> ________________________________________
> From: Vishal Gupta <gv...@gmail.com>
> Sent: Wednesday, December 21, 2016 9:19 PM
> To: dev@falcon.apache.org
> Subject: Re: Falcon issue after Kerberos implementation
>
>
> Cluster is HDP 2.3.2
>
> On Thu, Dec 22, 2016 at 1:17 PM, Vishal Gupta <gv...@gmail.com> wrote:
>
> > Hello Team,
> > We are implementing Kerberos at one the clusters.
> >
> > After Kerberos implementation looks like Falcon while using SPNEGO to
> > authenticate the user, is failing while authenticating and connecting to
> > Oozie.
> >
> >
> >
> > Any suggestions please. Details below..
> >
> >
> > 2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
> > 737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//
> > entities/delete/process/edmhdpif-oozie-icbdev-th] ~ Unable to reach
> > workflow engine for deletion or deletion failed
> (AbstractEntityManager:265)
> >
> > org.apache.falcon.FalconException: IO_ERROR : java.io.IOException: Error
> > while connecting Oozie server. No of retries = 1. Exception = null
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > findBundles(OozieWorkflowEngine.java:303)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > doBundleAction(OozieWorkflowEngine.java:377)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > doBundleAction(OozieWorkflowEngine.java:371)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > delete(OozieWorkflowEngine.java:355)
> >
> > at org.apache.falcon.resource.
> > AbstractEntityManager.delete(AbstractEntityManager.java:252)
> >
> > at org.apache.falcon.resource.ConfigSyncService.delete(
> > ConfigSyncService.java:62)
> >
> > at sun.reflect.GeneratedMethodAccessor88.invoke(Unknown
> > Source)
> >
> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> > DelegatingMethodAccessorImpl.java:43)
> >
> > at java.lang.reflect.Method.invoke(Method.java:498)
> >
> > at org.apache.falcon.resource.channel.IPCChannel.invoke(
> > IPCChannel.java:49)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityManagerProxy.
> > java:230)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$EntityProxy.execute(
> > SchedulableEntityManagerProxy.java:575)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.
> > java:219)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy.delete_aroundBody2(
> > SchedulableEntityManagerProxy.java:232)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$AjcClosure3.run(
> > SchedulableEntityManagerProxy.java:1)
> >
> > at org.aspectj.runtime.reflect.JoinPointImpl.proceed(
> > JoinPointImpl.java:149)
> >
> > at org.apache.falcon.aspect.AbstractFalconAspect.
> > logAroundMonitored(AbstractFalconAspect.java:51)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.
> > java:206)
> >
> > at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown
> > Source)
> >
> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> > DelegatingMethodAccessorImpl.java:43)
> >
> > at java.lang.reflect.Method.invoke(Method.java:498)
> >
>
> > at com.sun.jersey.spi.container.
> > JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
> >
> > at com.sun.jersey.server.impl.model.method.dispatch.
> > AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(
> > AbstractResourceMethodDispatchProvider.java:185)
> >
> > at com.sun.jersey.server.impl.model.method.dispatch.
> > ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.
> > java:75)
> >
> > at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.
> > accept(HttpMethodRule.java:288)
> >
> > at com.sun.jersey.server.impl.
> uri.rules.RightHandPathRule.
> > accept(RightHandPathRule.java:147)
> >
> > at com.sun.jersey.server.impl.
> uri.rules.ResourceClassRule.
> > accept(ResourceClassRule.java:108)
> >
> > at com.sun.jersey.server.impl.
> uri.rules.RightHandPathRule.
> > accept(RightHandPathRule.java:147)
> >
> > at com.sun.jersey.server.impl.uri.rules.
> > RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
> >
> > at com.sun.jersey.spi.container.
> > servlet.WebComponent.service(WebComponent.java:416)
> >
> > at com.sun.jersey.spi.container.
> servlet.ServletContainer.
> > service(ServletContainer.java:537)
> >
> > at com.sun.jersey.spi.container.
> servlet.ServletContainer.
> > service(ServletContainer.java:699)
> >
> > at javax.servlet.http.HttpServlet.service(
> > HttpServlet.java:820)
> >
> > at org.mortbay.jetty.servlet.ServletHolder.handle(
> > ServletHolder.java:511)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1221)
> >
> > at org.apache.falcon.security.FalconAuthorizationFilter.
> > doFilter(FalconAuthorizationFilter.java:106)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1212)
> >
> > at org.apache.falcon.security.
> > FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.
> java:184)
> >
> > at org.apache.hadoop.security.authentication.server.
> > AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
> >
> > at org.apache.hadoop.security.authentication.server.
> > AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
> >
> > at org.apache.falcon.security.
> FalconAuthenticationFilter.
> > doFilter(FalconAuthenticationFilter.java:193)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1212)
> >
> > at org.apache.falcon.security.
> FalconAuditFilter.doFilter(
> > FalconAuditFilter.java:64)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1212)
> >
> > at org.mortbay.jetty.servlet.ServletHandler.handle(
> > ServletHandler.java:399)
> >
> > at org.mortbay.jetty.security.SecurityHandler.handle(
> > SecurityHandler.java:216)
> >
> > at org.mortbay.jetty.servlet.SessionHandler.handle(
> > SessionHandler.java:182)
> >
> > at org.mortbay.jetty.handler.ContextHandler.handle(
> > ContextHandler.java:767)
> >
> > at org.mortbay.jetty.webapp.WebAppContext.handle(
> > WebAppContext.java:450)
> >
> > at org.mortbay.jetty.handler.HandlerWrapper.handle(
>
> > HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:326)
> >
> > at org.mortbay.jetty.HttpConnection.handleRequest(
> > HttpConnection.java:542)
> >
> > at org.mortbay.jetty.HttpConnection$RequestHandler.
> > headerComplete(HttpConnection.java:928)
> >
> > at org.mortbay.jetty.HttpParser.
> > parseNext(HttpParser.java:549)
> >
> > at org.mortbay.jetty.HttpParser.
> parseAvailable(HttpParser.
> > java:212)
> >
> > at org.mortbay.jetty.HttpConnection.handle(
> > HttpConnection.java:404)
> >
> > at org.mortbay.jetty.bio.SocketConnector$Connection.
> > run(SocketConnector.java:228)
> >
> > at org.mortbay.thread.QueuedThreadPool$PoolThread.
> > run(QueuedThreadPool.java:582)
> >
> > Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
> > server. No of retries = 1. Exception = null
> >
> > at org.apache.oozie.client.
> OozieClient.validateWSVersion(
> > OozieClient.java:374)
> >
> > at org.apache.oozie.client.OozieClient.createURL(
> > OozieClient.java:459)
> >
> > at org.apache.oozie.client.OozieClient.access$000(
> > OozieClient.java:80)
> >
> > at org.apache.oozie.client.OozieClient$ClientCallable.
> > call(OozieClient.java:555)
> >
> > at org.apache.oozie.client.
> OozieClient.getBundleJobsInfo(
> > OozieClient.java:2149)
> >
> > at org.apache.oozie.client.ProxyOozieClient.access$2901(
> > ProxyOozieClient.java:48)
> >
> > at org.apache.oozie.client.ProxyOozieClient$29.call(
> > ProxyOozieClient.java:598)
> >
> > at org.apache.oozie.client.ProxyOozieClient$29.call(
> > ProxyOozieClient.java:596)
> >
> > at org.apache.oozie.client.OozieClient.doAs(OozieClient.
> > java:244)
> >
> > at org.apache.oozie.client.ProxyOozieClient.
> > getBundleJobsInfo(ProxyOozieClient.java:596)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > findBundles(OozieWorkflowEngine.java:290)
> >
> > ... 61 more
> >
> > Caused by: java.io.IOException: Error while connecting Oozie server. No
> of
> > retries = 1. Exception = null
> >
> > at org.apache.oozie.client.retry.
> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
> >
> > at org.apache.oozie.client.OozieClient.
> > createRetryableConnection(OozieClient.java:504)
> >
> > at org.apache.oozie.client.OozieClient.
> > getSupportedProtocolVersions(OozieClient.java:384)
> >
> > at org.apache.oozie.client.
> OozieClient.validateWSVersion(
> > OozieClient.java:344)
> >
> > ... 71 more
> >
> > Caused by: java.lang.reflect.UndeclaredThrowableException
> >
> > at org.apache.hadoop.security.UserGroupInformation.doAs(
> > UserGroupInformation.java:1672)
> >
> > at org.apache.oozie.client.ProxyOozieClient.
> > createConnection(ProxyOozieClient.java:79)
> >
> > at org.apache.oozie.client.OozieClient$1.doExecute(
> > OozieClient.java:507)
> >
> > at org.apache.oozie.client.retry.
> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
> >
> > ... 74 more
> >
> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
> valid
> > credentials provided (Mechanism level: Failed to find any Kerberos tgt)
> >
> > at org.apache.oozie.client.AuthOozieClient.
> > createConnection(AuthOozieClient.java:136)
> >
> > at org.apache.oozie.client.ProxyOozieClient.access$001(
> > ProxyOozieClient.java:48)
> >
> > at org.apache.oozie.client.ProxyOozieClient$1.run(
> > ProxyOozieClient.java:81)
> >
> > at org.apache.oozie.client.ProxyOozieClient$1.run(
> > ProxyOozieClient.java:79)
> >
> > at java.security.AccessController.doPrivileged(Native
> > Method)
> >
> > at javax.security.auth.Subject.doAs(Subject.java:422)
> >
> > at org.apache.hadoop.security.UserGroupInformation.doAs(
> > UserGroupInformation.java:1657)
> >
> > ... 77 more
> >
> > Caused by: org.apache.hadoop.security.authentication.client.
> AuthenticationException:
> > GSSException: No valid credentials provided (Mechanism level: Failed to
> > find any Kerberos tgt)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
> >
> > at org.apache.hadoop.security.authentication.client.
> > AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
> >
> > at org.apache.oozie.client.AuthOozieClient.
> > createConnection(AuthOozieClient.java:127)
> >
> > ... 83 more
> >
> > Caused by: GSSException: No valid credentials provided (Mechanism level:
> > Failed to find any Kerberos tgt)
> >
> > at sun.security.jgss.krb5.Krb5InitCredential.
> getInstance(
> > Krb5InitCredential.java:147)
> >
> > at sun.security.jgss.krb5.Krb5MechFactory.
> > getCredentialElement(Krb5MechFactory.java:122)
> >
> > at sun.security.jgss.krb5.Krb5MechFactory.
> > getMechanismContext(Krb5MechFactory.java:187)
> >
> > at sun.security.jgss.GSSManagerImpl.getMechanismContext(
> > GSSManagerImpl.java:224)
> >
> > at sun.security.jgss.GSSContextImpl.initSecContext(
> > GSSContextImpl.java:212)
> >
> > at sun.security.jgss.GSSContextImpl.initSecContext(
> > GSSContextImpl.java:179)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
> >
> > at java.security.AccessController.doPrivileged(Native
> > Method)
> >
> > at javax.security.auth.Subject.doAs(Subject.java:422)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
> >
> > ... 86 more
> >
> >
> >
> >
> >
>
>
>
Re: Falcon issue after Kerberos implementation
Posted by Venkat Ranganathan <vr...@hortonworks.com>.
This is addressed as part of FALCON-2025. It is fixed in 0.10 release
Thanks
Venkat
From: Vishal Gupta <gv...@gmail.com>
Date: Wednesday, December 21, 2016 at 9:37 PM
To: "dev@falcon.apache.org" <de...@falcon.apache.org>, Venkat Ranganathan <vr...@hortonworks.com>
Subject: Re: Falcon issue after Kerberos implementation
Hello Venkat,
Thanks for quick response. Could you please guide to any contacts in HWX or any online server where we can find the patch / answer.
Do you know in which version this issue is being resolved.
Thanks in advance.
Regards,
Vishal.
On Thu, Dec 22, 2016 at 1:34 PM, Venkat Ranganathan <vr...@hortonworks.com>> wrote:
> Caused by: AUTHENTICATION : Could not authenticate, GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
This shows that the Falcon kerberos context become invalidated. There was a fix in later versions and patch should be available from HWX support for the older versions
Thanks
Venkat
________________________________________
From: Vishal Gupta <gv...@gmail.com>>
Sent: Wednesday, December 21, 2016 9:19 PM
To: dev@falcon.apache.org<ma...@falcon.apache.org>
Subject: Re: Falcon issue after Kerberos implementation
Cluster is HDP 2.3.2
On Thu, Dec 22, 2016 at 1:17 PM, Vishal Gupta <gv...@gmail.com>> wrote:
> Hello Team,
> We are implementing Kerberos at one the clusters.
>
> After Kerberos implementation looks like Falcon while using SPNEGO to
> authenticate the user, is failing while authenticating and connecting to
> Oozie.
>
>
>
> Any suggestions please. Details below..
>
>
> 2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
> 737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//
> entities/delete/process/edmhdpif-oozie-icbdev-th] ~ Unable to reach
> workflow engine for deletion or deletion failed (AbstractEntityManager:265)
>
> org.apache.falcon.FalconException: IO_ERROR : java.io.IOException: Error
> while connecting Oozie server. No of retries = 1. Exception = null
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> findBundles(OozieWorkflowEngine.java:303)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> doBundleAction(OozieWorkflowEngine.java:377)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> doBundleAction(OozieWorkflowEngine.java:371)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> delete(OozieWorkflowEngine.java:355)
>
> at org.apache.falcon.resource.
> AbstractEntityManager.delete(AbstractEntityManager.java:252)
>
> at org.apache.falcon.resource.ConfigSyncService.delete(
> ConfigSyncService.java:62)
>
> at sun.reflect.GeneratedMethodAccessor88.invoke(Unknown
> Source)
>
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> at org.apache.falcon.resource.channel.IPCChannel.invoke(
> IPCChannel.java:49)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityManagerProxy.
> java:230)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$EntityProxy.execute(
> SchedulableEntityManagerProxy.java:575)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.
> java:219)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy.delete_aroundBody2(
> SchedulableEntityManagerProxy.java:232)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$AjcClosure3.run(
> SchedulableEntityManagerProxy.java:1)
>
> at org.aspectj.runtime.reflect.JoinPointImpl.proceed(
> JoinPointImpl.java:149)
>
> at org.apache.falcon.aspect.AbstractFalconAspect.
> logAroundMonitored(AbstractFalconAspect.java:51)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.
> java:206)
>
> at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown
> Source)
>
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> at com.sun.jersey.spi.container.
> JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
>
> at com.sun.jersey.server.impl.model.method.dispatch.
> AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(
> AbstractResourceMethodDispatchProvider.java:185)
>
> at com.sun.jersey.server.impl.model.method.dispatch.
> ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.
> java:75)
>
> at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.
> accept(HttpMethodRule.java:288)
>
> at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.
> accept(RightHandPathRule.java:147)
>
> at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.
> accept(ResourceClassRule.java:108)
>
> at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.
> accept(RightHandPathRule.java:147)
>
> at com.sun.jersey.server.impl.uri.rules.
> RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
>
> at com.sun.jersey.spi.container.
> servlet.WebComponent.service(WebComponent.java:416)
>
> at com.sun.jersey.spi.container.servlet.ServletContainer.
> service(ServletContainer.java:537)
>
> at com.sun.jersey.spi.container.servlet.ServletContainer.
> service(ServletContainer.java:699)
>
> at javax.servlet.http.HttpServlet.service(
> HttpServlet.java:820)
>
> at org.mortbay.jetty.servlet.ServletHolder.handle(
> ServletHolder.java:511)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1221)
>
> at org.apache.falcon.security.FalconAuthorizationFilter.
> doFilter(FalconAuthorizationFilter.java:106)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.apache.falcon.security.
> FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:184)
>
> at org.apache.hadoop.security.authentication.server.
> AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
>
> at org.apache.hadoop.security.authentication.server.
> AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
>
> at org.apache.falcon.security.FalconAuthenticationFilter.
> doFilter(FalconAuthenticationFilter.java:193)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.apache.falcon.security.FalconAuditFilter.doFilter(
> FalconAuditFilter.java:64)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.mortbay.jetty.servlet.ServletHandler.handle(
> ServletHandler.java:399)
>
> at org.mortbay.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:216)
>
> at org.mortbay.jetty.servlet.SessionHandler.handle(
> SessionHandler.java:182)
>
> at org.mortbay.jetty.handler.ContextHandler.handle(
> ContextHandler.java:767)
>
> at org.mortbay.jetty.webapp.WebAppContext.handle(
> WebAppContext.java:450)
>
> at org.mortbay.jetty.handler.HandlerWrapper.handle(
> HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:326)
>
> at org.mortbay.jetty.HttpConnection.handleRequest(
> HttpConnection.java:542)
>
> at org.mortbay.jetty.HttpConnection$RequestHandler.
> headerComplete(HttpConnection.java:928)
>
> at org.mortbay.jetty.HttpParser.
> parseNext(HttpParser.java:549)
>
> at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.
> java:212)
>
> at org.mortbay.jetty.HttpConnection.handle(
> HttpConnection.java:404)
>
> at org.mortbay.jetty.bio.SocketConnector$Connection.
> run(SocketConnector.java:228)
>
> at org.mortbay.thread.QueuedThreadPool$PoolThread.
> run(QueuedThreadPool.java:582)
>
> Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
> server. No of retries = 1. Exception = null
>
> at org.apache.oozie.client.OozieClient.validateWSVersion(
> OozieClient.java:374)
>
> at org.apache.oozie.client.OozieClient.createURL(
> OozieClient.java:459)
>
> at org.apache.oozie.client.OozieClient.access$000(
> OozieClient.java:80)
>
> at org.apache.oozie.client.OozieClient$ClientCallable.
> call(OozieClient.java:555)
>
> at org.apache.oozie.client.OozieClient.getBundleJobsInfo(
> OozieClient.java:2149)
>
> at org.apache.oozie.client.ProxyOozieClient.access$2901(
> ProxyOozieClient.java:48)
>
> at org.apache.oozie.client.ProxyOozieClient$29.call(
> ProxyOozieClient.java:598)
>
> at org.apache.oozie.client.ProxyOozieClient$29.call(
> ProxyOozieClient.java:596)
>
> at org.apache.oozie.client.OozieClient.doAs(OozieClient.
> java:244)
>
> at org.apache.oozie.client.ProxyOozieClient.
> getBundleJobsInfo(ProxyOozieClient.java:596)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> findBundles(OozieWorkflowEngine.java:290)
>
> ... 61 more
>
> Caused by: java.io.IOException: Error while connecting Oozie server. No of
> retries = 1. Exception = null
>
> at org.apache.oozie.client.retry.
> ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
>
> at org.apache.oozie.client.OozieClient.
> createRetryableConnection(OozieClient.java:504)
>
> at org.apache.oozie.client.OozieClient.
> getSupportedProtocolVersions(OozieClient.java:384)
>
> at org.apache.oozie.client.OozieClient.validateWSVersion(
> OozieClient.java:344)
>
> ... 71 more
>
> Caused by: java.lang.reflect.UndeclaredThrowableException
>
> at org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1672)
>
> at org.apache.oozie.client.ProxyOozieClient.
> createConnection(ProxyOozieClient.java:79)
>
> at org.apache.oozie.client.OozieClient$1.doExecute(
> OozieClient.java:507)
>
> at org.apache.oozie.client.retry.
> ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
>
> ... 74 more
>
> Caused by: AUTHENTICATION : Could not authenticate, GSSException: No valid
> credentials provided (Mechanism level: Failed to find any Kerberos tgt)
>
> at org.apache.oozie.client.AuthOozieClient.
> createConnection(AuthOozieClient.java:136)
>
> at org.apache.oozie.client.ProxyOozieClient.access$001(
> ProxyOozieClient.java:48)
>
> at org.apache.oozie.client.ProxyOozieClient$1.run(
> ProxyOozieClient.java:81)
>
> at org.apache.oozie.client.ProxyOozieClient$1.run(
> ProxyOozieClient.java:79)
>
> at java.security.AccessController.doPrivileged(Native
> Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:422)
>
> at org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1657)
>
> ... 77 more
>
> Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException:
> GSSException: No valid credentials provided (Mechanism level: Failed to
> find any Kerberos tgt)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
>
> at org.apache.hadoop.security.authentication.client.
> AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
>
> at org.apache.oozie.client.AuthOozieClient.
> createConnection(AuthOozieClient.java:127)
>
> ... 83 more
>
> Caused by: GSSException: No valid credentials provided (Mechanism level:
> Failed to find any Kerberos tgt)
>
> at sun.security.jgss.krb5.Krb5InitCredential.getInstance(
> Krb5InitCredential.java:147)
>
> at sun.security.jgss.krb5.Krb5MechFactory.
> getCredentialElement(Krb5MechFactory.java:122)
>
> at sun.security.jgss.krb5.Krb5MechFactory.
> getMechanismContext(Krb5MechFactory.java:187)
>
> at sun.security.jgss.GSSManagerImpl.getMechanismContext(
> GSSManagerImpl.java:224)
>
> at sun.security.jgss.GSSContextImpl.initSecContext(
> GSSContextImpl.java:212)
>
> at sun.security.jgss.GSSContextImpl.initSecContext(
> GSSContextImpl.java:179)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
>
> at java.security.AccessController.doPrivileged(Native
> Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:422)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
>
> ... 86 more
>
>
>
>
>
Re: Falcon issue after Kerberos implementation
Posted by Vishal Gupta <gv...@gmail.com>.
Hello Venkat,
Thanks for quick response. Could you please guide to any contacts in HWX or
any online server where we can find the patch / answer.
Do you know in which version this issue is being resolved.
Thanks in advance.
Regards,
Vishal.
On Thu, Dec 22, 2016 at 1:34 PM, Venkat Ranganathan <
vranganathan@hortonworks.com> wrote:
> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
> valid credentials provided (Mechanism level: Failed to find any Kerberos
> tgt)
>
> This shows that the Falcon kerberos context become invalidated. There
> was a fix in later versions and patch should be available from HWX support
> for the older versions
>
> Thanks
> Venkat
> ________________________________________
> From: Vishal Gupta <gv...@gmail.com>
> Sent: Wednesday, December 21, 2016 9:19 PM
> To: dev@falcon.apache.org
> Subject: Re: Falcon issue after Kerberos implementation
>
> Cluster is HDP 2.3.2
>
> On Thu, Dec 22, 2016 at 1:17 PM, Vishal Gupta <gv...@gmail.com> wrote:
>
> > Hello Team,
> > We are implementing Kerberos at one the clusters.
> >
> > After Kerberos implementation looks like Falcon while using SPNEGO to
> > authenticate the user, is failing while authenticating and connecting to
> > Oozie.
> >
> >
> >
> > Any suggestions please. Details below..
> >
> >
> > 2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
> > 737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//
> > entities/delete/process/edmhdpif-oozie-icbdev-th] ~ Unable to reach
> > workflow engine for deletion or deletion failed
> (AbstractEntityManager:265)
> >
> > org.apache.falcon.FalconException: IO_ERROR : java.io.IOException: Error
> > while connecting Oozie server. No of retries = 1. Exception = null
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > findBundles(OozieWorkflowEngine.java:303)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > doBundleAction(OozieWorkflowEngine.java:377)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > doBundleAction(OozieWorkflowEngine.java:371)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > delete(OozieWorkflowEngine.java:355)
> >
> > at org.apache.falcon.resource.
> > AbstractEntityManager.delete(AbstractEntityManager.java:252)
> >
> > at org.apache.falcon.resource.ConfigSyncService.delete(
> > ConfigSyncService.java:62)
> >
> > at sun.reflect.GeneratedMethodAccessor88.invoke(Unknown
> > Source)
> >
> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> > DelegatingMethodAccessorImpl.java:43)
> >
> > at java.lang.reflect.Method.invoke(Method.java:498)
> >
> > at org.apache.falcon.resource.channel.IPCChannel.invoke(
> > IPCChannel.java:49)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityManagerProxy.
> > java:230)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$EntityProxy.execute(
> > SchedulableEntityManagerProxy.java:575)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.
> > java:219)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy.delete_aroundBody2(
> > SchedulableEntityManagerProxy.java:232)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy$AjcClosure3.run(
> > SchedulableEntityManagerProxy.java:1)
> >
> > at org.aspectj.runtime.reflect.JoinPointImpl.proceed(
> > JoinPointImpl.java:149)
> >
> > at org.apache.falcon.aspect.AbstractFalconAspect.
> > logAroundMonitored(AbstractFalconAspect.java:51)
> >
> > at org.apache.falcon.resource.proxy.
> > SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.
> > java:206)
> >
> > at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown
> > Source)
> >
> > at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> > DelegatingMethodAccessorImpl.java:43)
> >
> > at java.lang.reflect.Method.invoke(Method.java:498)
> >
> > at com.sun.jersey.spi.container.
> > JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
> >
> > at com.sun.jersey.server.impl.model.method.dispatch.
> > AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(
> > AbstractResourceMethodDispatchProvider.java:185)
> >
> > at com.sun.jersey.server.impl.model.method.dispatch.
> > ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.
> > java:75)
> >
> > at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.
> > accept(HttpMethodRule.java:288)
> >
> > at com.sun.jersey.server.impl.
> uri.rules.RightHandPathRule.
> > accept(RightHandPathRule.java:147)
> >
> > at com.sun.jersey.server.impl.
> uri.rules.ResourceClassRule.
> > accept(ResourceClassRule.java:108)
> >
> > at com.sun.jersey.server.impl.
> uri.rules.RightHandPathRule.
> > accept(RightHandPathRule.java:147)
> >
> > at com.sun.jersey.server.impl.uri.rules.
> > RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
> >
> > at com.sun.jersey.server.impl.application.
> > WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
> >
> > at com.sun.jersey.spi.container.
> > servlet.WebComponent.service(WebComponent.java:416)
> >
> > at com.sun.jersey.spi.container.
> servlet.ServletContainer.
> > service(ServletContainer.java:537)
> >
> > at com.sun.jersey.spi.container.
> servlet.ServletContainer.
> > service(ServletContainer.java:699)
> >
> > at javax.servlet.http.HttpServlet.service(
> > HttpServlet.java:820)
> >
> > at org.mortbay.jetty.servlet.ServletHolder.handle(
> > ServletHolder.java:511)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1221)
> >
> > at org.apache.falcon.security.FalconAuthorizationFilter.
> > doFilter(FalconAuthorizationFilter.java:106)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1212)
> >
> > at org.apache.falcon.security.
> > FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.
> java:184)
> >
> > at org.apache.hadoop.security.authentication.server.
> > AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
> >
> > at org.apache.hadoop.security.authentication.server.
> > AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
> >
> > at org.apache.falcon.security.
> FalconAuthenticationFilter.
> > doFilter(FalconAuthenticationFilter.java:193)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1212)
> >
> > at org.apache.falcon.security.
> FalconAuditFilter.doFilter(
> > FalconAuditFilter.java:64)
> >
> > at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> > doFilter(ServletHandler.java:1212)
> >
> > at org.mortbay.jetty.servlet.ServletHandler.handle(
> > ServletHandler.java:399)
> >
> > at org.mortbay.jetty.security.SecurityHandler.handle(
> > SecurityHandler.java:216)
> >
> > at org.mortbay.jetty.servlet.SessionHandler.handle(
> > SessionHandler.java:182)
> >
> > at org.mortbay.jetty.handler.ContextHandler.handle(
> > ContextHandler.java:767)
> >
> > at org.mortbay.jetty.webapp.WebAppContext.handle(
> > WebAppContext.java:450)
> >
> > at org.mortbay.jetty.handler.HandlerWrapper.handle(
> > HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:326)
> >
> > at org.mortbay.jetty.HttpConnection.handleRequest(
> > HttpConnection.java:542)
> >
> > at org.mortbay.jetty.HttpConnection$RequestHandler.
> > headerComplete(HttpConnection.java:928)
> >
> > at org.mortbay.jetty.HttpParser.
> > parseNext(HttpParser.java:549)
> >
> > at org.mortbay.jetty.HttpParser.
> parseAvailable(HttpParser.
> > java:212)
> >
> > at org.mortbay.jetty.HttpConnection.handle(
> > HttpConnection.java:404)
> >
> > at org.mortbay.jetty.bio.SocketConnector$Connection.
> > run(SocketConnector.java:228)
> >
> > at org.mortbay.thread.QueuedThreadPool$PoolThread.
> > run(QueuedThreadPool.java:582)
> >
> > Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
> > server. No of retries = 1. Exception = null
> >
> > at org.apache.oozie.client.
> OozieClient.validateWSVersion(
> > OozieClient.java:374)
> >
> > at org.apache.oozie.client.OozieClient.createURL(
> > OozieClient.java:459)
> >
> > at org.apache.oozie.client.OozieClient.access$000(
> > OozieClient.java:80)
> >
> > at org.apache.oozie.client.OozieClient$ClientCallable.
> > call(OozieClient.java:555)
> >
> > at org.apache.oozie.client.
> OozieClient.getBundleJobsInfo(
> > OozieClient.java:2149)
> >
> > at org.apache.oozie.client.ProxyOozieClient.access$2901(
> > ProxyOozieClient.java:48)
> >
> > at org.apache.oozie.client.ProxyOozieClient$29.call(
> > ProxyOozieClient.java:598)
> >
> > at org.apache.oozie.client.ProxyOozieClient$29.call(
> > ProxyOozieClient.java:596)
> >
> > at org.apache.oozie.client.OozieClient.doAs(OozieClient.
> > java:244)
> >
> > at org.apache.oozie.client.ProxyOozieClient.
> > getBundleJobsInfo(ProxyOozieClient.java:596)
> >
> > at org.apache.falcon.workflow.
> engine.OozieWorkflowEngine.
> > findBundles(OozieWorkflowEngine.java:290)
> >
> > ... 61 more
> >
> > Caused by: java.io.IOException: Error while connecting Oozie server. No
> of
> > retries = 1. Exception = null
> >
> > at org.apache.oozie.client.retry.
> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
> >
> > at org.apache.oozie.client.OozieClient.
> > createRetryableConnection(OozieClient.java:504)
> >
> > at org.apache.oozie.client.OozieClient.
> > getSupportedProtocolVersions(OozieClient.java:384)
> >
> > at org.apache.oozie.client.
> OozieClient.validateWSVersion(
> > OozieClient.java:344)
> >
> > ... 71 more
> >
> > Caused by: java.lang.reflect.UndeclaredThrowableException
> >
> > at org.apache.hadoop.security.UserGroupInformation.doAs(
> > UserGroupInformation.java:1672)
> >
> > at org.apache.oozie.client.ProxyOozieClient.
> > createConnection(ProxyOozieClient.java:79)
> >
> > at org.apache.oozie.client.OozieClient$1.doExecute(
> > OozieClient.java:507)
> >
> > at org.apache.oozie.client.retry.
> > ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
> >
> > ... 74 more
> >
> > Caused by: AUTHENTICATION : Could not authenticate, GSSException: No
> valid
> > credentials provided (Mechanism level: Failed to find any Kerberos tgt)
> >
> > at org.apache.oozie.client.AuthOozieClient.
> > createConnection(AuthOozieClient.java:136)
> >
> > at org.apache.oozie.client.ProxyOozieClient.access$001(
> > ProxyOozieClient.java:48)
> >
> > at org.apache.oozie.client.ProxyOozieClient$1.run(
> > ProxyOozieClient.java:81)
> >
> > at org.apache.oozie.client.ProxyOozieClient$1.run(
> > ProxyOozieClient.java:79)
> >
> > at java.security.AccessController.doPrivileged(Native
> > Method)
> >
> > at javax.security.auth.Subject.doAs(Subject.java:422)
> >
> > at org.apache.hadoop.security.UserGroupInformation.doAs(
> > UserGroupInformation.java:1657)
> >
> > ... 77 more
> >
> > Caused by: org.apache.hadoop.security.authentication.client.
> AuthenticationException:
> > GSSException: No valid credentials provided (Mechanism level: Failed to
> > find any Kerberos tgt)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
> >
> > at org.apache.hadoop.security.authentication.client.
> > AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
> >
> > at org.apache.oozie.client.AuthOozieClient.
> > createConnection(AuthOozieClient.java:127)
> >
> > ... 83 more
> >
> > Caused by: GSSException: No valid credentials provided (Mechanism level:
> > Failed to find any Kerberos tgt)
> >
> > at sun.security.jgss.krb5.Krb5InitCredential.
> getInstance(
> > Krb5InitCredential.java:147)
> >
> > at sun.security.jgss.krb5.Krb5MechFactory.
> > getCredentialElement(Krb5MechFactory.java:122)
> >
> > at sun.security.jgss.krb5.Krb5MechFactory.
> > getMechanismContext(Krb5MechFactory.java:187)
> >
> > at sun.security.jgss.GSSManagerImpl.getMechanismContext(
> > GSSManagerImpl.java:224)
> >
> > at sun.security.jgss.GSSContextImpl.initSecContext(
> > GSSContextImpl.java:212)
> >
> > at sun.security.jgss.GSSContextImpl.initSecContext(
> > GSSContextImpl.java:179)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
> >
> > at java.security.AccessController.doPrivileged(Native
> > Method)
> >
> > at javax.security.auth.Subject.doAs(Subject.java:422)
> >
> > at org.apache.hadoop.security.authentication.client.
> > KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
> >
> > ... 86 more
> >
> >
> >
> >
> >
>
Re: Falcon issue after Kerberos implementation
Posted by Venkat Ranganathan <vr...@hortonworks.com>.
> Caused by: AUTHENTICATION : Could not authenticate, GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
This shows that the Falcon kerberos context become invalidated. There was a fix in later versions and patch should be available from HWX support for the older versions
Thanks
Venkat
________________________________________
From: Vishal Gupta <gv...@gmail.com>
Sent: Wednesday, December 21, 2016 9:19 PM
To: dev@falcon.apache.org
Subject: Re: Falcon issue after Kerberos implementation
Cluster is HDP 2.3.2
On Thu, Dec 22, 2016 at 1:17 PM, Vishal Gupta <gv...@gmail.com> wrote:
> Hello Team,
> We are implementing Kerberos at one the clusters.
>
> After Kerberos implementation looks like Falcon while using SPNEGO to
> authenticate the user, is failing while authenticating and connecting to
> Oozie.
>
>
>
> Any suggestions please. Details below..
>
>
> 2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
> 737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//
> entities/delete/process/edmhdpif-oozie-icbdev-th] ~ Unable to reach
> workflow engine for deletion or deletion failed (AbstractEntityManager:265)
>
> org.apache.falcon.FalconException: IO_ERROR : java.io.IOException: Error
> while connecting Oozie server. No of retries = 1. Exception = null
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> findBundles(OozieWorkflowEngine.java:303)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> doBundleAction(OozieWorkflowEngine.java:377)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> doBundleAction(OozieWorkflowEngine.java:371)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> delete(OozieWorkflowEngine.java:355)
>
> at org.apache.falcon.resource.
> AbstractEntityManager.delete(AbstractEntityManager.java:252)
>
> at org.apache.falcon.resource.ConfigSyncService.delete(
> ConfigSyncService.java:62)
>
> at sun.reflect.GeneratedMethodAccessor88.invoke(Unknown
> Source)
>
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> at org.apache.falcon.resource.channel.IPCChannel.invoke(
> IPCChannel.java:49)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityManagerProxy.
> java:230)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$EntityProxy.execute(
> SchedulableEntityManagerProxy.java:575)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.
> java:219)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy.delete_aroundBody2(
> SchedulableEntityManagerProxy.java:232)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$AjcClosure3.run(
> SchedulableEntityManagerProxy.java:1)
>
> at org.aspectj.runtime.reflect.JoinPointImpl.proceed(
> JoinPointImpl.java:149)
>
> at org.apache.falcon.aspect.AbstractFalconAspect.
> logAroundMonitored(AbstractFalconAspect.java:51)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.
> java:206)
>
> at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown
> Source)
>
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> at com.sun.jersey.spi.container.
> JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
>
> at com.sun.jersey.server.impl.model.method.dispatch.
> AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(
> AbstractResourceMethodDispatchProvider.java:185)
>
> at com.sun.jersey.server.impl.model.method.dispatch.
> ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.
> java:75)
>
> at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.
> accept(HttpMethodRule.java:288)
>
> at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.
> accept(RightHandPathRule.java:147)
>
> at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.
> accept(ResourceClassRule.java:108)
>
> at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.
> accept(RightHandPathRule.java:147)
>
> at com.sun.jersey.server.impl.uri.rules.
> RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
>
> at com.sun.jersey.spi.container.
> servlet.WebComponent.service(WebComponent.java:416)
>
> at com.sun.jersey.spi.container.servlet.ServletContainer.
> service(ServletContainer.java:537)
>
> at com.sun.jersey.spi.container.servlet.ServletContainer.
> service(ServletContainer.java:699)
>
> at javax.servlet.http.HttpServlet.service(
> HttpServlet.java:820)
>
> at org.mortbay.jetty.servlet.ServletHolder.handle(
> ServletHolder.java:511)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1221)
>
> at org.apache.falcon.security.FalconAuthorizationFilter.
> doFilter(FalconAuthorizationFilter.java:106)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.apache.falcon.security.
> FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:184)
>
> at org.apache.hadoop.security.authentication.server.
> AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
>
> at org.apache.hadoop.security.authentication.server.
> AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
>
> at org.apache.falcon.security.FalconAuthenticationFilter.
> doFilter(FalconAuthenticationFilter.java:193)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.apache.falcon.security.FalconAuditFilter.doFilter(
> FalconAuditFilter.java:64)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.mortbay.jetty.servlet.ServletHandler.handle(
> ServletHandler.java:399)
>
> at org.mortbay.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:216)
>
> at org.mortbay.jetty.servlet.SessionHandler.handle(
> SessionHandler.java:182)
>
> at org.mortbay.jetty.handler.ContextHandler.handle(
> ContextHandler.java:767)
>
> at org.mortbay.jetty.webapp.WebAppContext.handle(
> WebAppContext.java:450)
>
> at org.mortbay.jetty.handler.HandlerWrapper.handle(
> HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:326)
>
> at org.mortbay.jetty.HttpConnection.handleRequest(
> HttpConnection.java:542)
>
> at org.mortbay.jetty.HttpConnection$RequestHandler.
> headerComplete(HttpConnection.java:928)
>
> at org.mortbay.jetty.HttpParser.
> parseNext(HttpParser.java:549)
>
> at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.
> java:212)
>
> at org.mortbay.jetty.HttpConnection.handle(
> HttpConnection.java:404)
>
> at org.mortbay.jetty.bio.SocketConnector$Connection.
> run(SocketConnector.java:228)
>
> at org.mortbay.thread.QueuedThreadPool$PoolThread.
> run(QueuedThreadPool.java:582)
>
> Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
> server. No of retries = 1. Exception = null
>
> at org.apache.oozie.client.OozieClient.validateWSVersion(
> OozieClient.java:374)
>
> at org.apache.oozie.client.OozieClient.createURL(
> OozieClient.java:459)
>
> at org.apache.oozie.client.OozieClient.access$000(
> OozieClient.java:80)
>
> at org.apache.oozie.client.OozieClient$ClientCallable.
> call(OozieClient.java:555)
>
> at org.apache.oozie.client.OozieClient.getBundleJobsInfo(
> OozieClient.java:2149)
>
> at org.apache.oozie.client.ProxyOozieClient.access$2901(
> ProxyOozieClient.java:48)
>
> at org.apache.oozie.client.ProxyOozieClient$29.call(
> ProxyOozieClient.java:598)
>
> at org.apache.oozie.client.ProxyOozieClient$29.call(
> ProxyOozieClient.java:596)
>
> at org.apache.oozie.client.OozieClient.doAs(OozieClient.
> java:244)
>
> at org.apache.oozie.client.ProxyOozieClient.
> getBundleJobsInfo(ProxyOozieClient.java:596)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> findBundles(OozieWorkflowEngine.java:290)
>
> ... 61 more
>
> Caused by: java.io.IOException: Error while connecting Oozie server. No of
> retries = 1. Exception = null
>
> at org.apache.oozie.client.retry.
> ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
>
> at org.apache.oozie.client.OozieClient.
> createRetryableConnection(OozieClient.java:504)
>
> at org.apache.oozie.client.OozieClient.
> getSupportedProtocolVersions(OozieClient.java:384)
>
> at org.apache.oozie.client.OozieClient.validateWSVersion(
> OozieClient.java:344)
>
> ... 71 more
>
> Caused by: java.lang.reflect.UndeclaredThrowableException
>
> at org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1672)
>
> at org.apache.oozie.client.ProxyOozieClient.
> createConnection(ProxyOozieClient.java:79)
>
> at org.apache.oozie.client.OozieClient$1.doExecute(
> OozieClient.java:507)
>
> at org.apache.oozie.client.retry.
> ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
>
> ... 74 more
>
> Caused by: AUTHENTICATION : Could not authenticate, GSSException: No valid
> credentials provided (Mechanism level: Failed to find any Kerberos tgt)
>
> at org.apache.oozie.client.AuthOozieClient.
> createConnection(AuthOozieClient.java:136)
>
> at org.apache.oozie.client.ProxyOozieClient.access$001(
> ProxyOozieClient.java:48)
>
> at org.apache.oozie.client.ProxyOozieClient$1.run(
> ProxyOozieClient.java:81)
>
> at org.apache.oozie.client.ProxyOozieClient$1.run(
> ProxyOozieClient.java:79)
>
> at java.security.AccessController.doPrivileged(Native
> Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:422)
>
> at org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1657)
>
> ... 77 more
>
> Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException:
> GSSException: No valid credentials provided (Mechanism level: Failed to
> find any Kerberos tgt)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
>
> at org.apache.hadoop.security.authentication.client.
> AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
>
> at org.apache.oozie.client.AuthOozieClient.
> createConnection(AuthOozieClient.java:127)
>
> ... 83 more
>
> Caused by: GSSException: No valid credentials provided (Mechanism level:
> Failed to find any Kerberos tgt)
>
> at sun.security.jgss.krb5.Krb5InitCredential.getInstance(
> Krb5InitCredential.java:147)
>
> at sun.security.jgss.krb5.Krb5MechFactory.
> getCredentialElement(Krb5MechFactory.java:122)
>
> at sun.security.jgss.krb5.Krb5MechFactory.
> getMechanismContext(Krb5MechFactory.java:187)
>
> at sun.security.jgss.GSSManagerImpl.getMechanismContext(
> GSSManagerImpl.java:224)
>
> at sun.security.jgss.GSSContextImpl.initSecContext(
> GSSContextImpl.java:212)
>
> at sun.security.jgss.GSSContextImpl.initSecContext(
> GSSContextImpl.java:179)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
>
> at java.security.AccessController.doPrivileged(Native
> Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:422)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
>
> ... 86 more
>
>
>
>
>
Re: Falcon issue after Kerberos implementation
Posted by Vishal Gupta <gv...@gmail.com>.
Cluster is HDP 2.3.2
On Thu, Dec 22, 2016 at 1:17 PM, Vishal Gupta <gv...@gmail.com> wrote:
> Hello Team,
> We are implementing Kerberos at one the clusters.
>
> After Kerberos implementation looks like Falcon while using SPNEGO to
> authenticate the user, is failing while authenticating and connecting to
> Oozie.
>
>
>
> Any suggestions please. Details below..
>
>
> 2016-12-21 07:52:42,597 ERROR - [1529601858@qtp-708533063-243 -
> 737162dd-6235-494c-a0b4-c8e132fc5491:ifrsdev:DELETE//
> entities/delete/process/edmhdpif-oozie-icbdev-th] ~ Unable to reach
> workflow engine for deletion or deletion failed (AbstractEntityManager:265)
>
> org.apache.falcon.FalconException: IO_ERROR : java.io.IOException: Error
> while connecting Oozie server. No of retries = 1. Exception = null
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> findBundles(OozieWorkflowEngine.java:303)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> doBundleAction(OozieWorkflowEngine.java:377)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> doBundleAction(OozieWorkflowEngine.java:371)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> delete(OozieWorkflowEngine.java:355)
>
> at org.apache.falcon.resource.
> AbstractEntityManager.delete(AbstractEntityManager.java:252)
>
> at org.apache.falcon.resource.ConfigSyncService.delete(
> ConfigSyncService.java:62)
>
> at sun.reflect.GeneratedMethodAccessor88.invoke(Unknown
> Source)
>
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> at org.apache.falcon.resource.channel.IPCChannel.invoke(
> IPCChannel.java:49)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$3.doExecute(SchedulableEntityManagerProxy.
> java:230)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$EntityProxy.execute(
> SchedulableEntityManagerProxy.java:575)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$3.execute(SchedulableEntityManagerProxy.
> java:219)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy.delete_aroundBody2(
> SchedulableEntityManagerProxy.java:232)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy$AjcClosure3.run(
> SchedulableEntityManagerProxy.java:1)
>
> at org.aspectj.runtime.reflect.JoinPointImpl.proceed(
> JoinPointImpl.java:149)
>
> at org.apache.falcon.aspect.AbstractFalconAspect.
> logAroundMonitored(AbstractFalconAspect.java:51)
>
> at org.apache.falcon.resource.proxy.
> SchedulableEntityManagerProxy.delete(SchedulableEntityManagerProxy.
> java:206)
>
> at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown
> Source)
>
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>
> at java.lang.reflect.Method.invoke(Method.java:498)
>
> at com.sun.jersey.spi.container.
> JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)
>
> at com.sun.jersey.server.impl.model.method.dispatch.
> AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(
> AbstractResourceMethodDispatchProvider.java:185)
>
> at com.sun.jersey.server.impl.model.method.dispatch.
> ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.
> java:75)
>
> at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.
> accept(HttpMethodRule.java:288)
>
> at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.
> accept(RightHandPathRule.java:147)
>
> at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.
> accept(ResourceClassRule.java:108)
>
> at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.
> accept(RightHandPathRule.java:147)
>
> at com.sun.jersey.server.impl.uri.rules.
> RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349)
>
> at com.sun.jersey.server.impl.application.
> WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339)
>
> at com.sun.jersey.spi.container.
> servlet.WebComponent.service(WebComponent.java:416)
>
> at com.sun.jersey.spi.container.servlet.ServletContainer.
> service(ServletContainer.java:537)
>
> at com.sun.jersey.spi.container.servlet.ServletContainer.
> service(ServletContainer.java:699)
>
> at javax.servlet.http.HttpServlet.service(
> HttpServlet.java:820)
>
> at org.mortbay.jetty.servlet.ServletHolder.handle(
> ServletHolder.java:511)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1221)
>
> at org.apache.falcon.security.FalconAuthorizationFilter.
> doFilter(FalconAuthorizationFilter.java:106)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.apache.falcon.security.
> FalconAuthenticationFilter$2.doFilter(FalconAuthenticationFilter.java:184)
>
> at org.apache.hadoop.security.authentication.server.
> AuthenticationFilter.doFilter(AuthenticationFilter.java:595)
>
> at org.apache.hadoop.security.authentication.server.
> AuthenticationFilter.doFilter(AuthenticationFilter.java:554)
>
> at org.apache.falcon.security.FalconAuthenticationFilter.
> doFilter(FalconAuthenticationFilter.java:193)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.apache.falcon.security.FalconAuditFilter.doFilter(
> FalconAuditFilter.java:64)
>
> at org.mortbay.jetty.servlet.ServletHandler$CachedChain.
> doFilter(ServletHandler.java:1212)
>
> at org.mortbay.jetty.servlet.ServletHandler.handle(
> ServletHandler.java:399)
>
> at org.mortbay.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:216)
>
> at org.mortbay.jetty.servlet.SessionHandler.handle(
> SessionHandler.java:182)
>
> at org.mortbay.jetty.handler.ContextHandler.handle(
> ContextHandler.java:767)
>
> at org.mortbay.jetty.webapp.WebAppContext.handle(
> WebAppContext.java:450)
>
> at org.mortbay.jetty.handler.HandlerWrapper.handle(
> HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:326)
>
> at org.mortbay.jetty.HttpConnection.handleRequest(
> HttpConnection.java:542)
>
> at org.mortbay.jetty.HttpConnection$RequestHandler.
> headerComplete(HttpConnection.java:928)
>
> at org.mortbay.jetty.HttpParser.
> parseNext(HttpParser.java:549)
>
> at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.
> java:212)
>
> at org.mortbay.jetty.HttpConnection.handle(
> HttpConnection.java:404)
>
> at org.mortbay.jetty.bio.SocketConnector$Connection.
> run(SocketConnector.java:228)
>
> at org.mortbay.thread.QueuedThreadPool$PoolThread.
> run(QueuedThreadPool.java:582)
>
> Caused by: IO_ERROR : java.io.IOException: Error while connecting Oozie
> server. No of retries = 1. Exception = null
>
> at org.apache.oozie.client.OozieClient.validateWSVersion(
> OozieClient.java:374)
>
> at org.apache.oozie.client.OozieClient.createURL(
> OozieClient.java:459)
>
> at org.apache.oozie.client.OozieClient.access$000(
> OozieClient.java:80)
>
> at org.apache.oozie.client.OozieClient$ClientCallable.
> call(OozieClient.java:555)
>
> at org.apache.oozie.client.OozieClient.getBundleJobsInfo(
> OozieClient.java:2149)
>
> at org.apache.oozie.client.ProxyOozieClient.access$2901(
> ProxyOozieClient.java:48)
>
> at org.apache.oozie.client.ProxyOozieClient$29.call(
> ProxyOozieClient.java:598)
>
> at org.apache.oozie.client.ProxyOozieClient$29.call(
> ProxyOozieClient.java:596)
>
> at org.apache.oozie.client.OozieClient.doAs(OozieClient.
> java:244)
>
> at org.apache.oozie.client.ProxyOozieClient.
> getBundleJobsInfo(ProxyOozieClient.java:596)
>
> at org.apache.falcon.workflow.engine.OozieWorkflowEngine.
> findBundles(OozieWorkflowEngine.java:290)
>
> ... 61 more
>
> Caused by: java.io.IOException: Error while connecting Oozie server. No of
> retries = 1. Exception = null
>
> at org.apache.oozie.client.retry.
> ConnectionRetriableClient.execute(ConnectionRetriableClient.java:66)
>
> at org.apache.oozie.client.OozieClient.
> createRetryableConnection(OozieClient.java:504)
>
> at org.apache.oozie.client.OozieClient.
> getSupportedProtocolVersions(OozieClient.java:384)
>
> at org.apache.oozie.client.OozieClient.validateWSVersion(
> OozieClient.java:344)
>
> ... 71 more
>
> Caused by: java.lang.reflect.UndeclaredThrowableException
>
> at org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1672)
>
> at org.apache.oozie.client.ProxyOozieClient.
> createConnection(ProxyOozieClient.java:79)
>
> at org.apache.oozie.client.OozieClient$1.doExecute(
> OozieClient.java:507)
>
> at org.apache.oozie.client.retry.
> ConnectionRetriableClient.execute(ConnectionRetriableClient.java:44)
>
> ... 74 more
>
> Caused by: AUTHENTICATION : Could not authenticate, GSSException: No valid
> credentials provided (Mechanism level: Failed to find any Kerberos tgt)
>
> at org.apache.oozie.client.AuthOozieClient.
> createConnection(AuthOozieClient.java:136)
>
> at org.apache.oozie.client.ProxyOozieClient.access$001(
> ProxyOozieClient.java:48)
>
> at org.apache.oozie.client.ProxyOozieClient$1.run(
> ProxyOozieClient.java:81)
>
> at org.apache.oozie.client.ProxyOozieClient$1.run(
> ProxyOozieClient.java:79)
>
> at java.security.AccessController.doPrivileged(Native
> Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:422)
>
> at org.apache.hadoop.security.UserGroupInformation.doAs(
> UserGroupInformation.java:1657)
>
> ... 77 more
>
> Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException:
> GSSException: No valid credentials provided (Mechanism level: Failed to
> find any Kerberos tgt)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:332)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.authenticate(KerberosAuthenticator.java:205)
>
> at org.apache.hadoop.security.authentication.client.
> AuthenticatedURL.openConnection(AuthenticatedURL.java:215)
>
> at org.apache.oozie.client.AuthOozieClient.
> createConnection(AuthOozieClient.java:127)
>
> ... 83 more
>
> Caused by: GSSException: No valid credentials provided (Mechanism level:
> Failed to find any Kerberos tgt)
>
> at sun.security.jgss.krb5.Krb5InitCredential.getInstance(
> Krb5InitCredential.java:147)
>
> at sun.security.jgss.krb5.Krb5MechFactory.
> getCredentialElement(Krb5MechFactory.java:122)
>
> at sun.security.jgss.krb5.Krb5MechFactory.
> getMechanismContext(Krb5MechFactory.java:187)
>
> at sun.security.jgss.GSSManagerImpl.getMechanismContext(
> GSSManagerImpl.java:224)
>
> at sun.security.jgss.GSSContextImpl.initSecContext(
> GSSContextImpl.java:212)
>
> at sun.security.jgss.GSSContextImpl.initSecContext(
> GSSContextImpl.java:179)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator$1.run(KerberosAuthenticator.java:311)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator$1.run(KerberosAuthenticator.java:287)
>
> at java.security.AccessController.doPrivileged(Native
> Method)
>
> at javax.security.auth.Subject.doAs(Subject.java:422)
>
> at org.apache.hadoop.security.authentication.client.
> KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:287)
>
> ... 86 more
>
>
>
>
>