How can I read two different hbase cluster with kerberos

[igyu <ig...@21cn.com>]

I have two hbase cluster and enable kerberos

I want run saprk application at clusterA to read clusterB with kerberos

in my code I add initKerberos functin like this

sparkSession.sparkContext.addFile("hdfs://clusterA/krb5ClusterB.conf")
sparkSession.sparkContext.addFile("hdfs://clusterA/ClusterB.keytab")

val krb5Path = SparkFiles.get("krb5ClusterB.conf")
val principal = config.getJSONObject("auth").getString("principal")
val keytab = SparkFiles.get("ClusterB.keytab")

System.setProperty("java.security.krb5.conf", krb5Path)

val conf = new Configuration
conf.set("hadoop.security.authentication", "Kerberos")
UserGroupInformation.setConfiguration(conf)
UserGroupInformation.loginUserFromKeytab(principal, keytab)

I want use this function to login ClusterB

at clusterA I run spark-submit --master yarn --deploy-mode cluster --principal ClusterA/spark@JOIN.COM --keytab /hadoop/app/ClusterA.keytab --class com.join.Synctool

but I get a error

 diagnostics: User class threw exception: org.apache.hadoop.security.KerberosAuthException: failure to login: for principal: jztwk/hadoop@JOIN.COM from keytab /hadoop/yarn/nm3/usercache/jztwk/appcache/application_1627287887991_1323/spark-9ac70772-d48f-43c3-814d-be210a5b33ea/userFiles-cfd44a8b-b252-4154-8876-1967e1609ec5/jztwk.keytab javax.security.auth.login.LoginException: Cannot locate KDC
at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1992)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytabAndReturnUGI(UserGroupInformation.java:1360)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:1140)
at com.join.hbase.reader.HbaseReader.initKerberos(HbaseReader.scala:203)
at com.join.hbase.reader.HbaseReader.beforeDo(HbaseReader.scala:138)
at com.join.Synctool$.main(Synctool.scala:327)
at com.join.Synctool.main(Synctool.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:673)
Caused by: javax.security.auth.login.LoginException: Cannot locate KDC
at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:808)
at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:617)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755)
at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:2070)
at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1982)
... 11 more
Caused by: KrbException: Cannot locate KDC
at sun.security.krb5.Config.getKDCList(Config.java:1121)
at sun.security.krb5.KdcComm.send(KdcComm.java:218)
at sun.security.krb5.KdcComm.send(KdcComm.java:200)
at sun.security.krb5.KrbAsReqBuilder.send(KrbAsReqBuilder.java:335)
at sun.security.krb5.KrbAsReqBuilder.action(KrbAsReqBuilder.java:488)
at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:780)
... 25 more
Caused by: KrbException: Generic error (description in e-text) (60) - Unable to locate KDC for realm JOIN.COM
at sun.security.krb5.Config.getKDCFromDNS(Config.java:1218)
at sun.security.krb5.Config.getKDCList(Config.java:1094)
... 30 more

ApplicationMaster host: bigdser5
ApplicationMaster RPC port: 34278
queue: root.users.jztwk
start time: 1629690007498
final status: FAILED
tracking URL: http://bigdser3:8088/proxy/application_1627287887991_1323/
user: jztwk
21/08/23 11:40:29 ERROR yarn.Client: Application diagnostics message: User class threw exception: org.apache.hadoop.security.KerberosAuthException: failure to login: for principal: jztwk/hadoop@JOIN.COM from keytab /hadoop/yarn/nm3/usercache/jztwk/appcache/application_1627287887991_1323/spark-9ac70772-d48f-43c3-814d-be210a5b33ea/userFiles-cfd44a8b-b252-4154-8876-1967e1609ec5/jztwk.keytab javax.security.auth.login.LoginException: Cannot locate KDC
at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1992)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytabAndReturnUGI(UserGroupInformation.java:1360)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:1140)
at com.join.hbase.reader.HbaseReader.initKerberos(HbaseReader.scala:203)
at com.join.hbase.reader.HbaseReader.beforeDo(HbaseReader.scala:138)
at com.join.Synctool$.main(Synctool.scala:327)
at com.join.Synctool.main(Synctool.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:673)
Caused by: javax.security.auth.login.LoginException: Cannot locate KDC
at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:808)
at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:617)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755)
at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:2070)
at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1982)
... 11 more
Caused by: KrbException: Cannot locate KDC
at sun.security.krb5.Config.getKDCList(Config.java:1121)
at sun.security.krb5.KdcComm.send(KdcComm.java:218)
at sun.security.krb5.KdcComm.send(KdcComm.java:200)
at sun.security.krb5.KrbAsReqBuilder.send(KrbAsReqBuilder.java:335)
at sun.security.krb5.KrbAsReqBuilder.action(KrbAsReqBuilder.java:488)
at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:780)
... 25 more
Caused by: KrbException: Generic error (description in e-text) (60) - Unable to locate KDC for realm JOIN.COM
at sun.security.krb5.Config.getKDCFromDNS(Config.java:1218)
at sun.security.krb5.Config.getKDCList(Config.java:1094)
... 30 more

Exception in thread "main" org.apache.spark.SparkException: Application application_1627287887991_1323 finished with failed status
at org.apache.spark.deploy.yarn.Client.run(Client.scala:1158)
at org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1606)
at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:851)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:926)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:935)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
21/08/23 11:40:29 INFO util.ShutdownHookManager: Shutdown hook called





igyu