You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by Bill Sanchez <bi...@gmail.com> on 2013/12/05 16:55:48 UTC
Socket Timeout in Secure Mode
I see the following exception when running in secure mode using Kerberos.
The exception appears to happen only under load and not consistently. It
seems to be somewhat similar to the bug below but the exceptions are
different. I am using 0.94.13.
Any advice is appreciated.
https://issues.apache.org/jira/browse/HBASE-7684
2013-12-05 06:28:36.529 GMT DEBUG [23805:hbase-tablepool-1-thread-1]
org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect - SASL
client context established. Negotiated QoP: auth-conf
2013-12-05 06:43:36.541 GMT WARN [23805:Thread-3] org.apache.hadoop.hbase.
client.HConnectionManager$HConnectionImplementation.processBatchCallback -
Failed all from region=appraisal,600,1386186300253.
2e7e692eddc816ee3ac8e405c3b949cb., hostname=plgrid-udn009.fanniemae.com,
port=60020
java.util.concurrent.ExecutionException: java.net.SocketTimeoutException:
Call to plgrid-udn009.fanniemae.com/10.198.99.9:60020 failed on socket
timeout exception: java.net.SocketTimeoutException: 900000 millis timeout
while waiting for channel to be ready for read. ch :
java.nio.channels.SocketChannel[connected
local=/10.198.99.13:59900 remote=
plgrid-udn009.fanniemae.com/10.198.99.9:60020]
at java.util.concurrent.FutureTask$Sync.innerGet(
FutureTask.java:222)
at java.util.concurrent.FutureTask.get(FutureTask.java:83)
at org.apache.hadoop.hbase.client.HConnectionManager$
HConnectionImplementation.processBatchCallback(HConnectionManager.java:1709)
at org.apache.hadoop.hbase.client.HConnectionManager$
HConnectionImplementation.processBatch(HConnectionManager.java:1561)
at org.apache.hadoop.hbase.client.HTable.flushCommits(
HTable.java:994)
at org.apache.hadoop.hbase.client.HTable.doPut(HTable.java:850)
at org.apache.hadoop.hbase.client.HTable.put(HTable.java:826)
at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$
TableRecordWriter.write(TableOutputFormat.java:123)
at org.apache.hadoop.hbase.mapreduce.TableOutputFormat$
TableRecordWriter.write(TableOutputFormat.java:84)
at com.platform.mapreduce.MRServiceJava.mapper.
DirectMapOutputWriter.write(DirectMapOutputWriter.java:55)
at org.apache.hadoop.mapreduce.TaskInputOutputContext.write(
TaskInputOutputContext.java:80)
at com.fanniemae.cram.hadoop.FileLoaderMapper.putDataInChunks(
FileLoaderMapper.java:354)
at com.fanniemae.cram.hadoop.FileLoaderMapper.map(
FileLoaderMapper.java:466)
at com.fanniemae.cram.hadoop.FileLoaderMapper.map(
FileLoaderMapper.java:1)
at org.apache.hadoop.mapreduce.Mapper.run_withoutCache(
Mapper.java:206)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:181)
at com.platform.mapreduce.MRServiceJava.mapper.MapTask.
runInternal(MapTask.java:262)
at com.platform.mapreduce.MRServiceJava.mapper.MapTask.
run(MapTask.java:109)
at com.platform.mapreduce.MRServiceJava.MRService$2.run(
MRService.java:579)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(
UserGroupInformation.java:1198)
at com.platform.mapreduce.work.adaptor.CredentialsAdaptor.
doAs(CredentialsAdaptor.java:232)
at com.platform.mapreduce.MRServiceJava.MRService.
onInvoke(MRService.java:573)
at com.platform.symphony.soam.ServiceContainer.invoke(
ServiceContainer.java:363)
at com.platform.symphony.soam.SoamCallbackWorkerThread.
nativeProcessNextCallbackEvent(Native Method)
at com.platform.symphony.soam.SoamCallbackWorkerThread.run(
SoamCallbackWorkerThread.java:77)
Caused by: java.net.SocketTimeoutException: Call to
plgrid-udn009.fanniemae.com/10.198.99.9:60020 failed on socket timeout
exception: java.net.SocketTimeoutException: 900000 millis timeout while
waiting for channel to be ready for read. ch :
java.nio.channels.SocketChannel[connected
local=/10.198.99.13:59900 remote=
plgrid-udn009.fanniemae.com/10.198.99.9:60020]
at org.apache.hadoop.hbase.ipc.HBaseClient.wrapException(
HBaseClient.java:1043)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.
java:1016)
at org.apache.hadoop.hbase.ipc.SecureRpcEngine$Invoker.
invoke(SecureRpcEngine.java:107)
at $Proxy5.multi(Unknown Source)
at org.apache.hadoop.hbase.client.HConnectionManager$
HConnectionImplementation$3$1.call(HConnectionManager.java:1538)
at org.apache.hadoop.hbase.client.HConnectionManager$
HConnectionImplementation$3$1.call(HConnectionManager.java:1536)
at org.apache.hadoop.hbase.client.ServerCallable.
withoutRetries(ServerCallable.java:229)
at org.apache.hadoop.hbase.client.HConnectionManager$
HConnectionImplementation$3.call(HConnectionManager.java:1545)
at org.apache.hadoop.hbase.client.HConnectionManager$
HConnectionImplementation$3.call(HConnectionManager.java:1533)
at java.util.concurrent.FutureTask$Sync.innerRun(
FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.
runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(
ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: java.net.SocketTimeoutException: 900000 millis timeout while
waiting for channel to be ready for read. ch :
java.nio.channels.SocketChannel[connected
local=/10.198.99.13:59900 remote=
plgrid-udn009.fanniemae.com/10.198.99.9:60020]
at org.apache.hadoop.net.SocketIOWithTimeout.doIO(
SocketIOWithTimeout.java:164)
at org.apache.hadoop.net.SocketInputStream.read(
SocketInputStream.java:155)
at org.apache.hadoop.net.SocketInputStream.read(
SocketInputStream.java:128)
at java.io.DataInputStream.readFully(DataInputStream.java:178)
at java.io.DataInputStream.readFully(DataInputStream.java:152)
at org.apache.hadoop.security.SaslInputStream.readMoreData(
SaslInputStream.java:88)
at org.apache.hadoop.security.SaslInputStream.read(
SaslInputStream.java:245)
at java.io.FilterInputStream.read(FilterInputStream.java:116)
at org.apache.hadoop.hbase.ipc.HBaseClient$Connection$
PingInputStream.read(HBaseClient.java:374)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
at java.io.DataInputStream.readInt(DataInputStream.java:370)
at org.apache.hadoop.hbase.ipc.SecureClient$SecureConnection.
receiveResponse(SecureClient.java:366)
at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(
HBaseClient.java:588)