You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@omid.apache.org by "Ohad Shacham (JIRA)" <ji...@apache.org> on 2018/02/27 10:06:00 UTC
[jira] [Resolved] (OMID-67) Avoid Kerberos logging multiple times
as it may cause race condition problems
[ https://issues.apache.org/jira/browse/OMID-67?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ohad Shacham resolved OMID-67.
------------------------------
Resolution: Fixed
> Avoid Kerberos logging multiple times as it may cause race condition problems
> -----------------------------------------------------------------------------
>
> Key: OMID-67
> URL: https://issues.apache.org/jira/browse/OMID-67
> Project: Apache Omid
> Issue Type: Bug
> Affects Versions: 0.8.2.0
> Reporter: Francisco Perez-Sorrosal
> Assignee: Francisco Perez-Sorrosal
> Priority: Major
> Fix For: 0.9.0.0
>
>
> Weird behaviour has been detected after the ticket expired when multiple logins to kerberos are triggered from different HBase modules in the TSO. The result is that the Kerberos ticket is not renewed and it causing this exception to be thrown:
> 2017-05-09 23:11:28,234 [tso-state-initializer] INFO TSOStateManagerImpl TSO Server state LWM 1780000000/Epoch 1780000000
> 2017-05-09 23:11:28,247 [tso-state-initializer] INFO LeaseManager TSO instance 10.215.69.146:54758 (Epoch 1780000000) advertised through ZK
> 2017-05-10 23:14:23,582 [htable-pool6-t269] WARN RpcClient Exception encountered while connecting to the server : javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
> 2017-05-10 23:14:23,583 [htable-pool6-t269] FATAL RpcClient SASL authentication failed. The most likely cause is missing or invalid credentials. Consider 'kinit'.
> javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
> at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
> at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:197)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection.setupSaslConnection(RpcClient.java:770)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection.access$600(RpcClient.java:357)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection$2.run(RpcClient.java:891)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection$2.run(RpcClient.java:888)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1637)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection.setupIOstreams(RpcClient.java:888)
> at org.apache.hadoop.hbase.ipc.RpcClient.getConnection(RpcClient.java:1543)
> at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1442)
> at org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1661)
> at org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1719)
> at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.multi(ClientProtos.java:30085)
> at org.apache.hadoop.hbase.client.MultiServerCallable.call(MultiServerCallable.java:113)
> at org.apache.hadoop.hbase.client.MultiServerCallable.call(MultiServerCallable.java:51)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:180)
> at org.apache.hadoop.hbase.client.AsyncProcess$1.run(AsyncProcess.java:543)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
> at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147)
> at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
> at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187)
> at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
> at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
> at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
> at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
> ... 23 more
> 2017-05-10 23:14:24,608 [htable-pool6-t269] WARN RpcClient Exception encountered while connecting to the server : javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
> 2017-05-10 23:14:24,608 [htable-pool6-t269] FATAL RpcClient SASL authentication failed. The most likely cause is missing or invalid credentials. Consider 'kinit'.
> javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
> at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
> at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:197)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection.setupSaslConnection(RpcClient.java:770)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection.access$600(RpcClient.java:357)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection$2.run(RpcClient.java:891)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection$2.run(RpcClient.java:888)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1637)
> at org.apache.hadoop.hbase.ipc.RpcClient$Connection.setupIOstreams(RpcClient.java:888)
> at org.apache.hadoop.hbase.ipc.RpcClient.getConnection(RpcClient.java:1543)
> at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1442)
> at org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1661)
> at org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1719)
> at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:30025)
> at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRowOrBefore(ProtobufUtil.java:1542)
> at org.apache.hadoop.hbase.client.HTable$2.call(HTable.java:710)
> at org.apache.hadoop.hbase.client.HTable$2.call(HTable.java:708)
> at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:114)
> at org.apache.hadoop.hbase.client.HTable.getRowOrBefore(HTable.java:714)
> at org.apache.hadoop.hbase.client.MetaScanner.metaScan(MetaScanner.java:144)
> at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.prefetchRegionCache(HConnectionManager.java:1220)
> at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:1284)
> at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:1160)
> at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:1114)
> at org.apache.hadoop.hbase.client.AsyncProcess.findDestLocation(AsyncProcess.java:365)
> at org.apache.hadoop.hbase.client.AsyncProcess.submit(AsyncProcess.java:507)
> at org.apache.hadoop.hbase.client.AsyncProcess.logAndResubmit(AsyncProcess.java:717)
> at org.apache.hadoop.hbase.client.AsyncProcess.receiveGlobalFailure(AsyncProcess.java:664)
> at org.apache.hadoop.hbase.client.AsyncProcess.access$100(AsyncProcess.java:93)
> at org.apache.hadoop.hbase.client.AsyncProcess$1.run(AsyncProcess.java:547)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
> at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147)
> at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
> at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187)
> at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
> at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
> at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
> at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
> ... 35 more
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)