You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hbase.apache.org by "Jaehui Lee (Jira)" <ji...@apache.org> on 2025/07/30 04:13:00 UTC
[jira] [Created] (HBASE-29482) Bulkload fails with viewfs authentication error
Jaehui Lee created HBASE-29482:
----------------------------------
Summary: Bulkload fails with viewfs authentication error
Key: HBASE-29482
URL: https://issues.apache.org/jira/browse/HBASE-29482
Project: HBase
Issue Type: Bug
Reporter: Jaehui Lee
Assignee: Jaehui Lee
When performing a bulkload operation where the HFiles are located on a federated HDFS and accessed via ViewFS paths, a Kerberos authentication error occurs.
{code:java}
2025-07-22T17:38:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=26,queue=2,port=16020] ipc.Client: Exception encountered while connecting to the server {host}
org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]
at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:179) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:399) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:578) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client$Connection.access$2100(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:799) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:795) ~[hadoop-common-3.4.1.jar:?]
at java.security.AccessController.doPrivileged(AccessController.java:714) ~[?:?]
at javax.security.auth.Subject.doAs(Subject.java:525) ~[?:?]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:795) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
at jdk.proxy2.$Proxy32.checkAccess(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$checkAccess$93(ClientNamenodeProtocolTranslatorPB.java:1464) ~[hadoop-hdfs-client-3.4.1.jar:?]
at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.checkAccess(ClientNamenodeProtocolTranslatorPB.java:1464) ~[hadoop-hdfs-client-3.4.1.jar:?]
at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:580) ~[?:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
at jdk.proxy2.$Proxy33.checkAccess(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.checkAccess(DFSClient.java:2987) ~[hadoop-hdfs-client-3.4.1.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem$66.doCall(DistributedFileSystem.java:3247) ~[hadoop-hdfs-client-3.4.1.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem$66.doCall(DistributedFileSystem.java:3244) ~[hadoop-hdfs-client-3.4.1.jar:?]
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem.access(DistributedFileSystem.java:3257) ~[hadoop-hdfs-client-3.4.1.jar:?]
at org.apache.hadoop.fs.FilterFileSystem.access(FilterFileSystem.java:473) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.fs.viewfs.ChRootedFileSystem.access(ChRootedFileSystem.java:256) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.fs.viewfs.ViewFileSystem.access(ViewFileSystem.java:581) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.hbase.regionserver.HStore.assertBulkLoadHFileOk(HStore.java:607) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:6737) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at java.security.AccessController.doPrivileged(AccessController.java:400) ~[?:?]
at javax.security.auth.Subject.doAs(Subject.java:453) ~[?:?]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?]
at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2443) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45008) ~[hbase-protocol-shaded-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:415) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3]
at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[hbase-server-2.5.11-hadoop3.jar:2.5.11-hadoop3] {code}
This happens because ViewFileSystem cannot generate delegation tokens properly for federated HDFS accessed through ViewFS paths. The simplest solution is to use the resolved HDFS paths instead of ViewFS paths when performing the bulkload.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)