You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by Shuja Rehman <sh...@gmail.com> on 2011/08/09 17:32:28 UTC
org.apache.hadoop.hbase.ZooKeeperConnectionException:
java.io.IOException: Too many open files
Hi All,
I am running map reduce jobs in queue and after some jobs, the following
exception starts to come. Let me know if I need to change any settings or
anything else
Thanks in advance.
11/08/09 06:19:56 INFO mapreduce.TableOutputFormat: Created table instance
for mytable
11/08/09 06:19:56 INFO zookeeper.ZooKeeper: Initiating client connection,
connectString=10.0.3.85:2181 sessionTimeout=180000 watcher=hconnection
11/08/09 06:19:56 ERROR mapreduce.TableInputFormat:
org.apache.hadoop.hbase.ZooKeeperConnectionException: java.io.IOException:
Too many open files
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:991)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.setupZookeeperTrackers(HConnectionManager.java:302)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.<init>(HConnectionManager.java:293)
at
org.apache.hadoop.hbase.client.HConnectionManager.getConnection(HConnectionManager.java:156)
at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:167)
at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:145)
at
org.apache.hadoop.hbase.mapreduce.TableInputFormat.setConf(TableInputFormat.java:91)
at
org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:62)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at
org.apache.hadoop.mapred.JobClient.writeNewSplits(JobClient.java:941)
at
org.apache.hadoop.mapred.JobClient.writeSplits(JobClient.java:961)
at org.apache.hadoop.mapred.JobClient.access$500(JobClient.java:170)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:880)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:833)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
at
org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:833)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:476)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:506)
at
hbaseaggregator.AggregatorRunner.runJob(AggregatorRunner.java:242)
at
hbaseaggregator.AggregatorRunner.Aggregator(AggregatorRunner.java:180)
at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:60)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
Caused by: java.io.IOException: Too many open files
at sun.nio.ch.IOUtil.initPipe(Native Method)
at sun.nio.ch.EPollSelectorImpl.<init>(EPollSelectorImpl.java:49)
at
sun.nio.ch.EPollSelectorProvider.openSelector(EPollSelectorProvider.java:18)
at java.nio.channels.Selector.open(Selector.java:209)
at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:160)
at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:331)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:377)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.connect(ZKUtil.java:97)
at
org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.<init>(ZooKeeperWatcher.java:119)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:989)
... 27 more
11/08/09 06:19:56 INFO mapred.JobClient: Cleaning up the staging area
hdfs://
hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0383
Exception in thread "main" java.lang.InternalError
at
sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:755)
at sun.misc.URLClassPath.getResource(URLClassPath.java:169)
at java.net.URLClassLoader$1.run(URLClassLoader.java:194)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at sun.misc.Launcher$ExtClassLoader.findClass(Launcher.java:229)
at java.lang.ClassLoader.loadClass(ClassLoader.java:307)
at java.lang.ClassLoader.loadClass(ClassLoader.java:296)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:248)
at
java.util.ResourceBundle$RBClassLoader.loadClass(ResourceBundle.java:435)
at
java.util.ResourceBundle$Control.newBundle(ResourceBundle.java:2289)
at java.util.ResourceBundle.loadBundle(ResourceBundle.java:1364)
at java.util.ResourceBundle.findBundle(ResourceBundle.java:1328)
at java.util.ResourceBundle.findBundle(ResourceBundle.java:1282)
at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:1224)
at java.util.ResourceBundle.getBundle(ResourceBundle.java:705)
at java.util.logging.Level.getLocalizedName(Level.java:223)
at java.util.logging.SimpleFormatter.format(SimpleFormatter.java:64)
at java.util.logging.StreamHandler.publish(StreamHandler.java:179)
at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:88)
at java.util.logging.Logger.log(Logger.java:458)
at java.util.logging.Logger.doLog(Logger.java:480)
at java.util.logging.Logger.log(Logger.java:569)
at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:72)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
Caused by: java.util.zip.ZipException: error in opening zip file
at java.util.zip.ZipFile.open(Native Method)
at java.util.zip.ZipFile.<init>(ZipFile.java:114)
at java.util.jar.JarFile.<init>(JarFile.java:135)
at java.util.jar.JarFile.<init>(JarFile.java:72)
at sun.misc.URLClassPath$JarLoader.getJarFile(URLClassPath.java:646)
at sun.misc.URLClassPath$JarLoader.access$600(URLClassPath.java:540)
at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:607)
at java.security.AccessController.doPrivileged(Native Method)
at sun.misc.URLClassPath$JarLoader.ensureOpen(URLClassPath.java:599)
at
sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:753)
... 29 more
--
Regards
Shuja-ur-Rehman Baig
<http://pk.linkedin.com/in/shujamughal>
Re: org.apache.hadoop.hbase.ZooKeeperConnectionException:
java.io.IOException: Too many open files
Posted by Stack <st...@duboce.net>.
You must not have set it everywhere. Its same issue as yesterday.
St.Ack
On Wed, Aug 10, 2011 at 6:43 AM, Shuja Rehman <sh...@gmail.com> wrote:
> Hi,
> I have set the limits as explained in the following link.
>
> https://ccp.cloudera.com/display/CDHDOC/HBase+Installation#HBaseInstallation-SettingUserLimitsforHBase
>
> Now it works fine for some more jobs But after that, I am getting following
> exception now.
>
> 11/08/10 06:33:07 INFO zookeeper.ZooKeeper: Initiating client connection,
> connectString=10.0.3.85:2181 sessionTimeout=180000 watcher=hconnection
> 11/08/10 06:33:07 INFO zookeeper.ClientCnxn: Opening socket connection to
> server /10.0.3.85:2181
> 11/08/10 06:33:07 INFO zookeeper.ClientCnxn: Socket connection established
> to hadoop.zoniversal.com/10.0.3.85:2181, initiating session
> 11/08/10 06:33:07 INFO zookeeper.ClientCnxn: Session establishment complete
> on server hadoop.zoniversal.com/10.0.3.85:2181, sessionid =
> 0x131628a4a072891, negotiated timeout = 40000
> 11/08/10 06:33:07 WARN mapred.JobClient: Use GenericOptionsParser for
> parsing the arguments. Applications should implement Tool for the same.
> 11/08/10 06:33:07 WARN hdfs.DFSClient: DFSOutputStream ResponseProcessor
> exception for block blk_156159297298887768_25457java.io.IOException: Too
> many open files
> at sun.nio.ch.IOUtil.initPipe(Native Method)
> at sun.nio.ch.EPollSelectorImpl.<init>(EPollSelectorImpl.java:49)
> at
> sun.nio.ch.EPollSelectorProvider.openSelector(EPollSelectorProvider.java:18)
> at
> org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.get(SocketIOWithTimeout.java:407)
> at
> org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:322)
> at
> org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
> at
> org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
> at
> org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
> at java.io.DataInputStream.readFully(DataInputStream.java:178)
> at java.io.DataInputStream.readLong(DataInputStream.java:399)
> at
> org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2638)
>
> 11/08/10 06:33:07 WARN hdfs.DFSClient: DataStreamer Exception:
> java.io.InterruptedIOException: Interruped while waiting for IO on channel
> java.nio.channels.SocketChannel[connected local=/10.0.3.85:57316 remote=/
> 10.0.3.85:50010]. 484999 millis timeout left.
> at
> org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:349)
> at
> org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
> at
> org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)
> at
> org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)
> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)
> at java.io.DataOutputStream.write(DataOutputStream.java:90)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2526)
>
> 11/08/10 06:33:07 INFO hdfs.DFSClient: Error Recovery for block
> blk_156159297298887768_25457 waiting for responder to exit.
> 11/08/10 06:33:07 WARN hdfs.DFSClient: Error Recovery for block
> blk_156159297298887768_25457 bad datanode[0] 10.0.3.85:50010
> 11/08/10 06:33:07 INFO mapred.JobClient: Cleaning up the staging area
> hdfs://
> hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0619
> Aug 10, 2011 6:33:08 AM hbaseaggregator.AggregatorDriver main
> SEVERE: null
> java.io.IOException: All datanodes 10.0.3.85:50010 are bad. Aborting...
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2774)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$1600(DFSClient.java:2305)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2477)
> 11/08/10 06:33:08 ERROR hdfs.DFSClient: Exception closing file
> /var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0619/libjars/zookeeper-3.3.3-cdh3u0.jar
> : java.io.IOException: All datanodes 10.0.3.85:50010 are bad. Aborting...
> java.io.IOException: All datanodes 10.0.3.85:50010 are bad. Aborting...
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2774)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$1600(DFSClient.java:2305)
> at
> org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2477)
>
>
> kindly let me know the solution.
> thnx
>
> On Tue, Aug 9, 2011 at 8:38 PM, Ian Varley <iv...@salesforce.com> wrote:
>
>> Hi Shuja,
>>
>> This question is mentioned in the HBase FAQ, here:
>>
>> http://wiki.apache.org/hadoop/Hbase/FAQ_Operations#A3
>>
>> which points to the HBase book:
>>
>> http://hbase.apache.org/book.html#ulimit
>>
>> "HBase is a database. It uses a lot of files all at the same time. The
>> default ulimit -n -- i.e. user file limit -- of 1024 on most *nix systems is
>> insufficient (On mac os x its 256). Any significant amount of loading will
>> lead you to "FAQ: Why do I see "java.io.IOException...(Too many open files)"
>> in my logs?" ... Do yourself a favor and change the upper bound on the
>> number of file descriptors. Set it to north of 10k. See the above referenced
>> FAQ for how. You should also up the hbase users' nproc setting; under load,
>> a low-nproc setting could manifest as OutOfMemoryError. To be clear, upping
>> the file descriptors and nproc for the user who is running the HBase process
>> is an operating system configuration, not an HBase configuration. Also, a
>> common mistake is that administrators will up the file descriptors for a
>> particular user but for whatever reason, HBase will be running as some one
>> else. HBase prints in its logs as the first line the ulimit its seeing.
>> Ensure its correct."
>>
>> - Ian
>>
>> On Aug 9, 2011, at 10:32 AM, Shuja Rehman wrote:
>>
>> Hi All,
>>
>> I am running map reduce jobs in queue and after some jobs, the following
>> exception starts to come. Let me know if I need to change any settings or
>> anything else
>>
>> Thanks in advance.
>>
>>
>> 11/08/09 06:19:56 INFO mapreduce.TableOutputFormat: Created table instance
>> for mytable
>> 11/08/09 06:19:56 INFO zookeeper.ZooKeeper: Initiating client connection,
>> connectString=10.0.3.85:2181 sessionTimeout=180000 watcher=hconnection
>> 11/08/09 06:19:56 ERROR mapreduce.TableInputFormat:
>> org.apache.hadoop.hbase.ZooKeeperConnectionException: java.io.IOException:
>> Too many open files
>> at
>>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:991)
>> at
>>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.setupZookeeperTrackers(HConnectionManager.java:302)
>> at
>>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.<init>(HConnectionManager.java:293)
>> at
>>
>> org.apache.hadoop.hbase.client.HConnectionManager.getConnection(HConnectionManager.java:156)
>> at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:167)
>> at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:145)
>> at
>>
>> org.apache.hadoop.hbase.mapreduce.TableInputFormat.setConf(TableInputFormat.java:91)
>> at
>> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:62)
>> at
>>
>> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
>> at
>> org.apache.hadoop.mapred.JobClient.writeNewSplits(JobClient.java:941)
>> at
>> org.apache.hadoop.mapred.JobClient.writeSplits(JobClient.java:961)
>> at org.apache.hadoop.mapred.JobClient.access$500(JobClient.java:170)
>> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:880)
>> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:833)
>> at java.security.AccessController.doPrivileged(Native Method)
>> at javax.security.auth.Subject.doAs(Subject.java:396)
>> at
>>
>> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
>> at
>> org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:833)
>> at org.apache.hadoop.mapreduce.Job.submit(Job.java:476)
>> at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:506)
>> at
>> hbaseaggregator.AggregatorRunner.runJob(AggregatorRunner.java:242)
>> at
>> hbaseaggregator.AggregatorRunner.Aggregator(AggregatorRunner.java:180)
>> at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:60)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at
>>
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> at
>>
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> at java.lang.reflect.Method.invoke(Method.java:597)
>> at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
>> Caused by: java.io.IOException: Too many open files
>> at sun.nio.ch.IOUtil.initPipe(Native Method)
>> at sun.nio.ch.EPollSelectorImpl.<init>(EPollSelectorImpl.java:49)
>> at
>>
>> sun.nio.ch.EPollSelectorProvider.openSelector(EPollSelectorProvider.java:18)
>> at java.nio.channels.Selector.open(Selector.java:209)
>> at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:160)
>> at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:331)
>> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:377)
>> at org.apache.hadoop.hbase.zookeeper.ZKUtil.connect(ZKUtil.java:97)
>> at
>>
>> org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.<init>(ZooKeeperWatcher.java:119)
>> at
>>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:989)
>> ... 27 more
>>
>> 11/08/09 06:19:56 INFO mapred.JobClient: Cleaning up the staging area
>> hdfs://
>>
>> hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0383
>> <
>> http://hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0383
>> >
>> Exception in thread "main" java.lang.InternalError
>> at
>> sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:755)
>> at sun.misc.URLClassPath.getResource(URLClassPath.java:169)
>> at java.net.URLClassLoader$1.run(URLClassLoader.java:194)
>> at java.security.AccessController.doPrivileged(Native Method)
>> at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
>> at sun.misc.Launcher$ExtClassLoader.findClass(Launcher.java:229)
>> at java.lang.ClassLoader.loadClass(ClassLoader.java:307)
>> at java.lang.ClassLoader.loadClass(ClassLoader.java:296)
>> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
>> at java.lang.ClassLoader.loadClass(ClassLoader.java:248)
>> at
>> java.util.ResourceBundle$RBClassLoader.loadClass(ResourceBundle.java:435)
>> at
>> java.util.ResourceBundle$Control.newBundle(ResourceBundle.java:2289)
>> at java.util.ResourceBundle.loadBundle(ResourceBundle.java:1364)
>> at java.util.ResourceBundle.findBundle(ResourceBundle.java:1328)
>> at java.util.ResourceBundle.findBundle(ResourceBundle.java:1282)
>> at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:1224)
>> at java.util.ResourceBundle.getBundle(ResourceBundle.java:705)
>> at java.util.logging.Level.getLocalizedName(Level.java:223)
>> at java.util.logging.SimpleFormatter.format(SimpleFormatter.java:64)
>> at java.util.logging.StreamHandler.publish(StreamHandler.java:179)
>> at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:88)
>> at java.util.logging.Logger.log(Logger.java:458)
>> at java.util.logging.Logger.doLog(Logger.java:480)
>> at java.util.logging.Logger.log(Logger.java:569)
>> at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:72)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at
>>
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>> at
>>
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>> at java.lang.reflect.Method.invoke(Method.java:597)
>> at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
>> Caused by: java.util.zip.ZipException: error in opening zip file
>> at java.util.zip.ZipFile.open(Native Method)
>> at java.util.zip.ZipFile.<init>(ZipFile.java:114)
>> at java.util.jar.JarFile.<init>(JarFile.java:135)
>> at java.util.jar.JarFile.<init>(JarFile.java:72)
>> at sun.misc.URLClassPath$JarLoader.getJarFile(URLClassPath.java:646)
>> at sun.misc.URLClassPath$JarLoader.access$600(URLClassPath.java:540)
>> at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:607)
>> at java.security.AccessController.doPrivileged(Native Method)
>> at sun.misc.URLClassPath$JarLoader.ensureOpen(URLClassPath.java:599)
>> at
>> sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:753)
>> ... 29 more
>>
>> --
>> Regards
>> Shuja-ur-Rehman Baig
>> <http://pk.linkedin.com/in/shujamughal>
>>
>>
>
>
> --
> Regards
> Shuja-ur-Rehman Baig
> <http://pk.linkedin.com/in/shujamughal>
>
Re: org.apache.hadoop.hbase.ZooKeeperConnectionException:
java.io.IOException: Too many open files
Posted by Shuja Rehman <sh...@gmail.com>.
Hi,
I have set the limits as explained in the following link.
https://ccp.cloudera.com/display/CDHDOC/HBase+Installation#HBaseInstallation-SettingUserLimitsforHBase
Now it works fine for some more jobs But after that, I am getting following
exception now.
11/08/10 06:33:07 INFO zookeeper.ZooKeeper: Initiating client connection,
connectString=10.0.3.85:2181 sessionTimeout=180000 watcher=hconnection
11/08/10 06:33:07 INFO zookeeper.ClientCnxn: Opening socket connection to
server /10.0.3.85:2181
11/08/10 06:33:07 INFO zookeeper.ClientCnxn: Socket connection established
to hadoop.zoniversal.com/10.0.3.85:2181, initiating session
11/08/10 06:33:07 INFO zookeeper.ClientCnxn: Session establishment complete
on server hadoop.zoniversal.com/10.0.3.85:2181, sessionid =
0x131628a4a072891, negotiated timeout = 40000
11/08/10 06:33:07 WARN mapred.JobClient: Use GenericOptionsParser for
parsing the arguments. Applications should implement Tool for the same.
11/08/10 06:33:07 WARN hdfs.DFSClient: DFSOutputStream ResponseProcessor
exception for block blk_156159297298887768_25457java.io.IOException: Too
many open files
at sun.nio.ch.IOUtil.initPipe(Native Method)
at sun.nio.ch.EPollSelectorImpl.<init>(EPollSelectorImpl.java:49)
at
sun.nio.ch.EPollSelectorProvider.openSelector(EPollSelectorProvider.java:18)
at
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.get(SocketIOWithTimeout.java:407)
at
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:322)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
at
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
at
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
at java.io.DataInputStream.readFully(DataInputStream.java:178)
at java.io.DataInputStream.readLong(DataInputStream.java:399)
at
org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2638)
11/08/10 06:33:07 WARN hdfs.DFSClient: DataStreamer Exception:
java.io.InterruptedIOException: Interruped while waiting for IO on channel
java.nio.channels.SocketChannel[connected local=/10.0.3.85:57316 remote=/
10.0.3.85:50010]. 484999 millis timeout left.
at
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:349)
at
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:146)
at
org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:107)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:105)
at java.io.DataOutputStream.write(DataOutputStream.java:90)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2526)
11/08/10 06:33:07 INFO hdfs.DFSClient: Error Recovery for block
blk_156159297298887768_25457 waiting for responder to exit.
11/08/10 06:33:07 WARN hdfs.DFSClient: Error Recovery for block
blk_156159297298887768_25457 bad datanode[0] 10.0.3.85:50010
11/08/10 06:33:07 INFO mapred.JobClient: Cleaning up the staging area
hdfs://
hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0619
Aug 10, 2011 6:33:08 AM hbaseaggregator.AggregatorDriver main
SEVERE: null
java.io.IOException: All datanodes 10.0.3.85:50010 are bad. Aborting...
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2774)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$1600(DFSClient.java:2305)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2477)
11/08/10 06:33:08 ERROR hdfs.DFSClient: Exception closing file
/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0619/libjars/zookeeper-3.3.3-cdh3u0.jar
: java.io.IOException: All datanodes 10.0.3.85:50010 are bad. Aborting...
java.io.IOException: All datanodes 10.0.3.85:50010 are bad. Aborting...
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2774)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$1600(DFSClient.java:2305)
at
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2477)
kindly let me know the solution.
thnx
On Tue, Aug 9, 2011 at 8:38 PM, Ian Varley <iv...@salesforce.com> wrote:
> Hi Shuja,
>
> This question is mentioned in the HBase FAQ, here:
>
> http://wiki.apache.org/hadoop/Hbase/FAQ_Operations#A3
>
> which points to the HBase book:
>
> http://hbase.apache.org/book.html#ulimit
>
> "HBase is a database. It uses a lot of files all at the same time. The
> default ulimit -n -- i.e. user file limit -- of 1024 on most *nix systems is
> insufficient (On mac os x its 256). Any significant amount of loading will
> lead you to "FAQ: Why do I see "java.io.IOException...(Too many open files)"
> in my logs?" ... Do yourself a favor and change the upper bound on the
> number of file descriptors. Set it to north of 10k. See the above referenced
> FAQ for how. You should also up the hbase users' nproc setting; under load,
> a low-nproc setting could manifest as OutOfMemoryError. To be clear, upping
> the file descriptors and nproc for the user who is running the HBase process
> is an operating system configuration, not an HBase configuration. Also, a
> common mistake is that administrators will up the file descriptors for a
> particular user but for whatever reason, HBase will be running as some one
> else. HBase prints in its logs as the first line the ulimit its seeing.
> Ensure its correct."
>
> - Ian
>
> On Aug 9, 2011, at 10:32 AM, Shuja Rehman wrote:
>
> Hi All,
>
> I am running map reduce jobs in queue and after some jobs, the following
> exception starts to come. Let me know if I need to change any settings or
> anything else
>
> Thanks in advance.
>
>
> 11/08/09 06:19:56 INFO mapreduce.TableOutputFormat: Created table instance
> for mytable
> 11/08/09 06:19:56 INFO zookeeper.ZooKeeper: Initiating client connection,
> connectString=10.0.3.85:2181 sessionTimeout=180000 watcher=hconnection
> 11/08/09 06:19:56 ERROR mapreduce.TableInputFormat:
> org.apache.hadoop.hbase.ZooKeeperConnectionException: java.io.IOException:
> Too many open files
> at
>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:991)
> at
>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.setupZookeeperTrackers(HConnectionManager.java:302)
> at
>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.<init>(HConnectionManager.java:293)
> at
>
> org.apache.hadoop.hbase.client.HConnectionManager.getConnection(HConnectionManager.java:156)
> at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:167)
> at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:145)
> at
>
> org.apache.hadoop.hbase.mapreduce.TableInputFormat.setConf(TableInputFormat.java:91)
> at
> org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:62)
> at
>
> org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
> at
> org.apache.hadoop.mapred.JobClient.writeNewSplits(JobClient.java:941)
> at
> org.apache.hadoop.mapred.JobClient.writeSplits(JobClient.java:961)
> at org.apache.hadoop.mapred.JobClient.access$500(JobClient.java:170)
> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:880)
> at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:833)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
>
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
> at
> org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:833)
> at org.apache.hadoop.mapreduce.Job.submit(Job.java:476)
> at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:506)
> at
> hbaseaggregator.AggregatorRunner.runJob(AggregatorRunner.java:242)
> at
> hbaseaggregator.AggregatorRunner.Aggregator(AggregatorRunner.java:180)
> at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:60)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
> Caused by: java.io.IOException: Too many open files
> at sun.nio.ch.IOUtil.initPipe(Native Method)
> at sun.nio.ch.EPollSelectorImpl.<init>(EPollSelectorImpl.java:49)
> at
>
> sun.nio.ch.EPollSelectorProvider.openSelector(EPollSelectorProvider.java:18)
> at java.nio.channels.Selector.open(Selector.java:209)
> at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:160)
> at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:331)
> at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:377)
> at org.apache.hadoop.hbase.zookeeper.ZKUtil.connect(ZKUtil.java:97)
> at
>
> org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.<init>(ZooKeeperWatcher.java:119)
> at
>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:989)
> ... 27 more
>
> 11/08/09 06:19:56 INFO mapred.JobClient: Cleaning up the staging area
> hdfs://
>
> hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0383
> <
> http://hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0383
> >
> Exception in thread "main" java.lang.InternalError
> at
> sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:755)
> at sun.misc.URLClassPath.getResource(URLClassPath.java:169)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:194)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
> at sun.misc.Launcher$ExtClassLoader.findClass(Launcher.java:229)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:307)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:296)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:248)
> at
> java.util.ResourceBundle$RBClassLoader.loadClass(ResourceBundle.java:435)
> at
> java.util.ResourceBundle$Control.newBundle(ResourceBundle.java:2289)
> at java.util.ResourceBundle.loadBundle(ResourceBundle.java:1364)
> at java.util.ResourceBundle.findBundle(ResourceBundle.java:1328)
> at java.util.ResourceBundle.findBundle(ResourceBundle.java:1282)
> at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:1224)
> at java.util.ResourceBundle.getBundle(ResourceBundle.java:705)
> at java.util.logging.Level.getLocalizedName(Level.java:223)
> at java.util.logging.SimpleFormatter.format(SimpleFormatter.java:64)
> at java.util.logging.StreamHandler.publish(StreamHandler.java:179)
> at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:88)
> at java.util.logging.Logger.log(Logger.java:458)
> at java.util.logging.Logger.doLog(Logger.java:480)
> at java.util.logging.Logger.log(Logger.java:569)
> at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:72)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
> Caused by: java.util.zip.ZipException: error in opening zip file
> at java.util.zip.ZipFile.open(Native Method)
> at java.util.zip.ZipFile.<init>(ZipFile.java:114)
> at java.util.jar.JarFile.<init>(JarFile.java:135)
> at java.util.jar.JarFile.<init>(JarFile.java:72)
> at sun.misc.URLClassPath$JarLoader.getJarFile(URLClassPath.java:646)
> at sun.misc.URLClassPath$JarLoader.access$600(URLClassPath.java:540)
> at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:607)
> at java.security.AccessController.doPrivileged(Native Method)
> at sun.misc.URLClassPath$JarLoader.ensureOpen(URLClassPath.java:599)
> at
> sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:753)
> ... 29 more
>
> --
> Regards
> Shuja-ur-Rehman Baig
> <http://pk.linkedin.com/in/shujamughal>
>
>
--
Regards
Shuja-ur-Rehman Baig
<http://pk.linkedin.com/in/shujamughal>
Re: org.apache.hadoop.hbase.ZooKeeperConnectionException:
java.io.IOException: Too many open files
Posted by Ian Varley <iv...@salesforce.com>.
Hi Shuja,
This question is mentioned in the HBase FAQ, here:
http://wiki.apache.org/hadoop/Hbase/FAQ_Operations#A3
which points to the HBase book:
http://hbase.apache.org/book.html#ulimit
"HBase is a database. It uses a lot of files all at the same time. The default ulimit -n -- i.e. user file limit -- of 1024 on most *nix systems is insufficient (On mac os x its 256). Any significant amount of loading will lead you to "FAQ: Why do I see "java.io.IOException...(Too many open files)" in my logs?" ... Do yourself a favor and change the upper bound on the number of file descriptors. Set it to north of 10k. See the above referenced FAQ for how. You should also up the hbase users' nproc setting; under load, a low-nproc setting could manifest as OutOfMemoryError. To be clear, upping the file descriptors and nproc for the user who is running the HBase process is an operating system configuration, not an HBase configuration. Also, a common mistake is that administrators will up the file descriptors for a particular user but for whatever reason, HBase will be running as some one else. HBase prints in its logs as the first line the ulimit its seeing. Ensure its correct."
- Ian
On Aug 9, 2011, at 10:32 AM, Shuja Rehman wrote:
Hi All,
I am running map reduce jobs in queue and after some jobs, the following
exception starts to come. Let me know if I need to change any settings or
anything else
Thanks in advance.
11/08/09 06:19:56 INFO mapreduce.TableOutputFormat: Created table instance
for mytable
11/08/09 06:19:56 INFO zookeeper.ZooKeeper: Initiating client connection,
connectString=10.0.3.85:2181 sessionTimeout=180000 watcher=hconnection
11/08/09 06:19:56 ERROR mapreduce.TableInputFormat:
org.apache.hadoop.hbase.ZooKeeperConnectionException: java.io.IOException:
Too many open files
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:991)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.setupZookeeperTrackers(HConnectionManager.java:302)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.<init>(HConnectionManager.java:293)
at
org.apache.hadoop.hbase.client.HConnectionManager.getConnection(HConnectionManager.java:156)
at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:167)
at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:145)
at
org.apache.hadoop.hbase.mapreduce.TableInputFormat.setConf(TableInputFormat.java:91)
at
org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:62)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
at
org.apache.hadoop.mapred.JobClient.writeNewSplits(JobClient.java:941)
at
org.apache.hadoop.mapred.JobClient.writeSplits(JobClient.java:961)
at org.apache.hadoop.mapred.JobClient.access$500(JobClient.java:170)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:880)
at org.apache.hadoop.mapred.JobClient$2.run(JobClient.java:833)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
at
org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:833)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:476)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:506)
at
hbaseaggregator.AggregatorRunner.runJob(AggregatorRunner.java:242)
at
hbaseaggregator.AggregatorRunner.Aggregator(AggregatorRunner.java:180)
at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:60)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
Caused by: java.io.IOException: Too many open files
at sun.nio.ch.IOUtil.initPipe(Native Method)
at sun.nio.ch.EPollSelectorImpl.<init>(EPollSelectorImpl.java:49)
at
sun.nio.ch.EPollSelectorProvider.openSelector(EPollSelectorProvider.java:18)
at java.nio.channels.Selector.open(Selector.java:209)
at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:160)
at org.apache.zookeeper.ClientCnxn.<init>(ClientCnxn.java:331)
at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:377)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.connect(ZKUtil.java:97)
at
org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.<init>(ZooKeeperWatcher.java:119)
at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getZooKeeperWatcher(HConnectionManager.java:989)
... 27 more
11/08/09 06:19:56 INFO mapred.JobClient: Cleaning up the staging area
hdfs://
hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0383<http://hadoop.zoniversal.com/var/lib/hadoop-0.20/cache/mapred/mapred/staging/root/.staging/job_201107251124_0383>
Exception in thread "main" java.lang.InternalError
at
sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:755)
at sun.misc.URLClassPath.getResource(URLClassPath.java:169)
at java.net.URLClassLoader$1.run(URLClassLoader.java:194)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
at sun.misc.Launcher$ExtClassLoader.findClass(Launcher.java:229)
at java.lang.ClassLoader.loadClass(ClassLoader.java:307)
at java.lang.ClassLoader.loadClass(ClassLoader.java:296)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
at java.lang.ClassLoader.loadClass(ClassLoader.java:248)
at
java.util.ResourceBundle$RBClassLoader.loadClass(ResourceBundle.java:435)
at
java.util.ResourceBundle$Control.newBundle(ResourceBundle.java:2289)
at java.util.ResourceBundle.loadBundle(ResourceBundle.java:1364)
at java.util.ResourceBundle.findBundle(ResourceBundle.java:1328)
at java.util.ResourceBundle.findBundle(ResourceBundle.java:1282)
at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:1224)
at java.util.ResourceBundle.getBundle(ResourceBundle.java:705)
at java.util.logging.Level.getLocalizedName(Level.java:223)
at java.util.logging.SimpleFormatter.format(SimpleFormatter.java:64)
at java.util.logging.StreamHandler.publish(StreamHandler.java:179)
at java.util.logging.ConsoleHandler.publish(ConsoleHandler.java:88)
at java.util.logging.Logger.log(Logger.java:458)
at java.util.logging.Logger.doLog(Logger.java:480)
at java.util.logging.Logger.log(Logger.java:569)
at hbaseaggregator.AggregatorDriver.main(AggregatorDriver.java:72)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:186)
Caused by: java.util.zip.ZipException: error in opening zip file
at java.util.zip.ZipFile.open(Native Method)
at java.util.zip.ZipFile.<init>(ZipFile.java:114)
at java.util.jar.JarFile.<init>(JarFile.java:135)
at java.util.jar.JarFile.<init>(JarFile.java:72)
at sun.misc.URLClassPath$JarLoader.getJarFile(URLClassPath.java:646)
at sun.misc.URLClassPath$JarLoader.access$600(URLClassPath.java:540)
at sun.misc.URLClassPath$JarLoader$1.run(URLClassPath.java:607)
at java.security.AccessController.doPrivileged(Native Method)
at sun.misc.URLClassPath$JarLoader.ensureOpen(URLClassPath.java:599)
at
sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:753)
... 29 more
--
Regards
Shuja-ur-Rehman Baig
<http://pk.linkedin.com/in/shujamughal>