You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-user@hadoop.apache.org by "Peng, Wei" <We...@xerox.com> on 2011/10/21 09:47:39 UTC
jobtracker cannot be started
Hi,
When I was running a job on hadoop with 75% mappers finished, the
jobtracker hung so that I cannot access
jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
well.
Then I stopped jobtracker and restarted it. However, the jobtracker
cannot be started. I received error message from jobtracker.log.out
saying
Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java heap
space
at
java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
at
java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
at
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
at
org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
at org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
at org.apache.hadoop.ipc.Client.call(Client.java:719)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
at $Proxy4.renewLease(Unknown Source)
at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
cationHandler.java:82)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
Handler.java:59)
at $Proxy4.renewLease(Unknown Source)
at
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
at
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
at java.lang.Thread.run(Thread.java:619)
Exception in thread "expireTrackers" java.lang.OutOfMemoryError: Java
heap space
at java.util.Arrays.copyOf(Arrays.java:2882)
at
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
a:100)
at
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
at java.lang.StringBuffer.append(StringBuffer.java:224)
at org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
at
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
a:1354)
at
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
2)
at
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
va:849)
at
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
3)
at
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
)
at
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
48)
at java.lang.Thread.run(Thread.java:619)
Exception in thread "IPC Server listener on 9001"
java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)
at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)
at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:324)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at org.mortbay.thread.Timeout.tick(Timeout.java:159)
at
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
va:587)
at
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
at
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
or.java:124)
at
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
707)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:189)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
Java HotSpot(TM) 64-Bit Server VM warning: Exception
java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
handler- the VM may need to be forcibly terminated
Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java heap
space
java.lang.reflect.InvocationTargetException
Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
.java:281)
at
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
er.java:44)
at
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
va:213)
at java.util.TimerThread.mainLoop(Timer.java:512)
at java.util.TimerThread.run(Timer.java:462)
Caused by: java.lang.OutOfMemoryError: Java heap space
Exception in thread "IPC Server Responder" java.lang.OutOfMemoryError:
Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
429)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
Exception in thread "expireLaunchingTasks" java.lang.OutOfMemoryError:
Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)
at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)
at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:324)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:189)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
429)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)
at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)
at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:324)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:181)
at
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
at
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)
at
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
at
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
at
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)
at
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:324)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
a:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
Impl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
at org.mortbay.log.Log.warn(Log.java:189)
at
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
at
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
at
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
at
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
at
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
409)
at
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Caused by: java.lang.OutOfMemoryError: Java heap space
I have no idea how to solve this problem. The memory on the jobtracker
server seems ok
total used free shared buffers
cached
Mem: 7148044 3578024 3570020 0 673768
467340
-/+ buffers/cache: 2436916 4711128
Swap: 1020116 160 1019956
Can someone help me on this?
Thanks a lot
Wei
RE: jobtracker cannot be started
Posted by "Peng, Wei" <We...@xerox.com>.
Sorry that I was out of town for these days and could not access
internet.
I have set the heap size to be 3000m, and restarted the job tracker
again.
However, the job tracker still cannot be started without any error
messages shown. The jobtracker webUI and "hadoop job -list all" also
hung without error messages. The jobtracker logs
hadoop-hadoop-jobtracker-*.local.log and
hadoop-hadoop-jobtracker-*.local.out do not show any messages either.
The blocksize and buffer size are default.
The content of limits.conf is
hadoop soft nofile 65535
hadoop hard nofile 65535
I have been working on this for long time, and have no clue what is
happening!
Wei
-----Original Message-----
From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
Sent: Friday, October 21, 2011 3:40 AM
To: common-user@hadoop.apache.org
Subject: Re: jobtracker cannot be started
How large is the blocksize? Hadoop uses a buffer size of 4kb for IO
operations, I can imagine that the balancer take a lot of memory. Check
if
they running. Also a good idea will be to increase the space (add more
datanodes). Check the fs.trash.interval setting or not (no settings
disable
the trash-facility).
also you can use "# hadoop dfs -expunge"
And, at least, check the limits on the server (limits.conf)
- alex
On Fri, Oct 21, 2011 at 12:09 PM, Peng, Wei <We...@xerox.com> wrote:
> Alex, thank you a lot for helping me. I will figure out how to change
> the conf file. It seems that even chattr -i does not work.
> Just one last question, why restarting the jobtracker needs such a big
> heap size? I had no problem to restart it before the jobtracker hung ?
> One problem of this hadoop cluster that I did not mention is the DFS
> space only has 5% left.
>
> Thanks
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 5:01 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> should, yes ;) I use 2000 in our environment, but depends on the
memory
> on
> your servers.
>
> regards,
> Alex
>
> On Fri, Oct 21, 2011 at 10:58 AM, Peng, Wei <We...@xerox.com>
wrote:
>
> > Yes, the heap size the default 1000m. /bin/java -Xmx1000m
> > So if I can change the heapsize to be bigger, I should be able to
> solve
> > this problem?
> >
> > Thanks,
> > Wei
> >
> > -----Original Message-----
> > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > Sent: Friday, October 21, 2011 4:53 AM
> > To: common-user@hadoop.apache.org
> > Subject: Re: jobtracker cannot be started
> >
> > looks like that the Heap utilization has exceeded the value set by
> -Xmx.
> > Do
> > a ps waux|grep java @jobtracker |grep -i xmx
> > The heapsize will be set in hadoop-env.sh:
> > export HADOOP_HEAPSIZE=
> >
> > default 1000, I think.
> >
> > - alex
> >
> > On Fri, Oct 21, 2011 at 10:31 AM, Peng, Wei <We...@xerox.com>
> wrote:
> >
> > > Thank you for your quick reply!!
> > >
> > > I cannot change the hadoop conf files because they are owned by a
> > person
> > > who has left the company, though I have the root access. My Java
> > version
> > > is java version "1.5.0_07"
> > > Java(TM) 2 Runtime Environment, Standard Edition (build
> 1.5.0_07-b03)
> > > Java HotSpot(TM) 64-Bit Server VM (build 1.5.0_07-b03, mixed mode)
> > >
> > > The log on http://jobtracker:50030/stacks is
> > > Process Thread Dump:
> > > 26 active threads
> > > Thread 53 (1424598978@qtp0-5):
> > > State: RUNNABLE
> > > Blocked count: 0
> > > Waited count: 29
> > > Stack:
> > > sun.management.ThreadImpl.getThreadInfo0(Native Method)
> > > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
> > > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
> > >
> > >
> >
>
org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.j
> > > ava:149)
> > >
> > >
> >
>
org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:505
> > > )
> > > javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> > > javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> > >
> > >
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
> > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)
> > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > Thread 43 (Trash Emptier):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 183
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > > org.apache.hadoop.fs.Trash$Emptier.run(Trash.java:234)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 36 (IPC Server handler 9 on 9000):
> > > State: WAITING
> > > Blocked count: 32
> > > Waited count: 3444
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 35 (IPC Server handler 8 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3446
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 34 (IPC Server handler 7 on 9000):
> > > State: WAITING
> > > Blocked count: 30
> > > Waited count: 3451
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 33 (IPC Server handler 6 on 9000):
> > > State: WAITING
> > > Blocked count: 33
> > > Waited count: 3445
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 32 (IPC Server handler 5 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3448
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 31 (IPC Server handler 4 on 9000):
> > > State: WAITING
> > > Blocked count: 33
> > > Waited count: 3446
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 30 (IPC Server handler 3 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3450
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 29 (IPC Server handler 2 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3449
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 28 (IPC Server handler 1 on 9000):
> > > State: WAITING
> > > Blocked count: 36
> > > Waited count: 3445
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 27 (IPC Server handler 0 on 9000):
> > > State: WAITING
> > > Blocked count: 44
> > > Waited count: 3445
> > > Waiting on
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > >
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 12 (IPC Server listener on 9000):
> > > State: RUNNABLE
> > > Blocked count: 5
> > > Waited count: 0
> > > Stack:
> > > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > >
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
> > > org.apache.hadoop.ipc.Server$Listener.run(Server.java:318)
> > > Thread 13 (IPC Server Responder):
> > > State: RUNNABLE
> > > Blocked count: 1
> > > Waited count: 0
> > > Stack:
> > > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > >
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > > org.apache.hadoop.ipc.Server$Responder.run(Server.java:478)
> > > Thread 26 (Timer-0):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 190
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.util.TimerThread.mainLoop(Timer.java:509)
> > > java.util.TimerThread.run(Timer.java:462)
> > > Thread 25 (1947740113@qtp0-1 - Acceptor0
> > > SelectChannelConnector@penguin33:50070):
> > > State: RUNNABLE
> > > Blocked count: 579
> > > Waited count: 5
> > > Stack:
> > > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > >
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > >
> > >
> >
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > va:429)
> > >
> > >
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > >
> > >
> >
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > or.java:124)
> > >
> > >
> >
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > 707)
> > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > > Thread 22
> > >
> >
>
(org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@67c7
> > > 980c):
> > > State: TIMED_WAITING
> > > Blocked count: 5
> > > Waited count: 127
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
>
org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(D
> > > ecommissionManager.java:65)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 21
> > >
> >
>
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@
> > > 2094257f):
> > > State: TIMED_WAITING
> > > Blocked count: 20
> > > Waited count: 1263
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
>
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.r
> > > un(FSNamesystem.java:2304)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 20
> > >
> >
>
(org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a51127a):
> > > State: TIMED_WAITING
> > > Blocked count: 21
> > > Waited count: 1875
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
>
org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseMan
> > > ager.java:349)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 19
> > >
> >
>
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@61
> > > 578aab):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 13
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
>
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run
> > > (FSNamesystem.java:2286)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 18
> > >
> >
>
(org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$Pending
> > > ReplicationMonitor@2339e351):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 13
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
>
org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingR
> > > eplicationMonitor.run(PendingReplicationBlocks.java:186)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 9 (RMI TCP Accept-0):
> > > State: RUNNABLE
> > > Blocked count: 0
> > > Waited count: 0
> > > Stack:
> > > java.net.PlainSocketImpl.socketAccept(Native Method)
> > > java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
> > > java.net.ServerSocket.implAccept(ServerSocket.java:453)
> > > java.net.ServerSocket.accept(ServerSocket.java:421)
> > >
> > >
> >
>
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTrans
> > > port.java:369)
> > >
> > >
> >
>
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 4 (Signal Dispatcher):
> > > State: RUNNABLE
> > > Blocked count: 0
> > > Waited count: 0
> > > Stack:
> > > Thread 3 (Finalizer):
> > > State: WAITING
> > > Blocked count: 0
> > > Waited count: 40
> > > Waiting on java.lang.ref.ReferenceQueue$Lock@22f62eba
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
> > > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
> > > java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
> > > Thread 2 (Reference Handler):
> > > State: WAITING
> > > Blocked count: 1
> > > Waited count: 39
> > > Waiting on java.lang.ref.Reference$Lock@646d6aa0
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.lang.Object.wait(Object.java:485)
> > >
java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
> > > Thread 1 (main):
> > > State: WAITING
> > > Blocked count: 10
> > > Waited count: 10
> > > Waiting on org.apache.hadoop.ipc.RPC$Server@41f6321
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.lang.Object.wait(Object.java:485)
> > > org.apache.hadoop.ipc.Server.join(Server.java:1122)
> > >
> > >
> >
>
org.apache.hadoop.hdfs.server.namenode.NameNode.join(NameNode.java:292)
> > >
> > >
> >
>
org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:966)
> > >
> > > Wei
> > >
> > > -----Original Message-----
> > > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > > Sent: Friday, October 21, 2011 4:15 AM
> > > To: common-user@hadoop.apache.org
> > > Subject: Re: jobtracker cannot be started
> > >
> > > add into hadoop-env.sh the opts to the jdk-call. The logs should
be
> > > accessible at he hadoop-log-directory.
> > >
> > > Also check http://jobtracker:50030/stacks - thats the same as
jstack
> > > (jstack
> > > PID). Also you can use jstack -F PID to get a corefile (similar to
> > > /stacks I
> > > think) @jobtracker.
> > >
> > > Are you using 64bit-JDK? Which version?
> > >
> > > regards,
> > > Alex
> > >
> > > On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com>
> > wrote:
> > >
> > > > I am using the default heap size, which is 1000MB. The
jobtracker
> > hung
> > > > when only I was running one job. Now I could not even restart
the
> > > > jobtracker.
> > > > Can you teach me how to turn on GC logging in hadoop?
> > > >
> > > > Thanks!
> > > > Wei
> > > >
> > > > -----Original Message-----
> > > > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > > > Sent: Friday, October 21, 2011 3:54 AM
> > > > To: common-user@hadoop.apache.org
> > > > Subject: Re: jobtracker cannot be started
> > > >
> > > > Hi,
> > > >
> > > > what are the heap size you given at the jobtracker? And how much
> > jobs
> > > /
> > > > users / tasks are run? What say a log?
> > > > Turn on GC logging:
> > > >
> >
http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
> > > >
> > > > - Alex
> > > >
> > > >
> > > > On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com>
> > wrote:
> > > >
> > > > > Hi,
> > > > >
> > > > >
> > > > >
> > > > > When I was running a job on hadoop with 75% mappers finished,
> the
> > > > > jobtracker hung so that I cannot access
> > > > > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status
hung
> > as
> > > > > well.
> > > > >
> > > > >
> > > > >
> > > > > Then I stopped jobtracker and restarted it. However, the
> > jobtracker
> > > > > cannot be started. I received error message from
> > jobtracker.log.out
> > > > > saying
> > > > >
> > > > >
> > > > >
> > > > > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError:
> > Java
> > > > heap
> > > > > space
> > > > >
> > > > > at
> > > > >
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> > > > >
> > > > > at
> > > > >
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> > > > >
> > > > > at
> > > > >
> > org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> > > > >
> > > > > at
> > > org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> > > > >
> > > > > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> > > > >
> > > > > at
org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> > > > >
> > > > > at $Proxy4.renewLease(Unknown Source)
> > > > >
> > > > > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
> > > Source)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > > > > cationHandler.java:82)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > > > > Handler.java:59)
> > > > >
> > > > > at $Proxy4.renewLease(Unknown Source)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> > > > >
> > > > > at
> > > > >
> > >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> > > > >
> > > > > at java.lang.Thread.run(Thread.java:619)
> > > > >
> > > > > Exception in thread "expireTrackers"
java.lang.OutOfMemoryError:
> > > Java
> > > > > heap space
> > > > >
> > > > > at java.util.Arrays.copyOf(Arrays.java:2882)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > > > > a:100)
> > > > >
> > > > > at
> > > > >
> > >
> java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> > > > >
> > > > > at java.lang.StringBuffer.append(StringBuffer.java:224)
> > > > >
> > > > > at
> > > org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > > > > a:1354)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > > > > 2)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > > > > va:849)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > > > > 3)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > > > > )
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > > > > 48)
> > > > >
> > > > > at java.lang.Thread.run(Thread.java:619)
> > > > >
> > > > > Exception in thread "IPC Server listener on 9001"
> > > > > java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > > > va:587)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > > > or.java:124)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > > > 707)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > > > > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM
> to
> > > > > handler- the VM may need to be forcibly terminated
> > > > >
> > > > > Exception in thread "retireJobs" java.lang.OutOfMemoryError:
> Java
> > > heap
> > > > > space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > Exception in thread "1586500830@qtp0-37"
> > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > Exception in thread "1513282332@qtp0-29"
> > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > > > > .java:281)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > > > > er.java:44)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > > > > va:213)
> > > > >
> > > > > at java.util.TimerThread.mainLoop(Timer.java:512)
> > > > >
> > > > > at java.util.TimerThread.run(Timer.java:462)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > Exception in thread "IPC Server Responder"
> > > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 429)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > Exception in thread "expireLaunchingTasks"
> > > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 429)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > >
org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > >
> > > > >
> > > > > I have no idea how to solve this problem. The memory on the
> > > jobtracker
> > > > > server seems ok
> > > > >
> > > > > total used free shared buffers
> > > > > cached
> > > > >
> > > > > Mem: 7148044 3578024 3570020 0 673768
> > > > > 467340
> > > > >
> > > > > -/+ buffers/cache: 2436916 4711128
> > > > >
> > > > > Swap: 1020116 160 1019956
> > > > >
> > > > >
> > > > >
> > > > > Can someone help me on this?
> > > > >
> > > > >
> > > > >
> > > > > Thanks a lot
> > > > >
> > > > > Wei
> > > > >
> > > > >
> > > >
> > > >
> > > > --
> > > > Alexander Lorenz
> > > > http://mapredit.blogspot.com
> > > >
> > >
> > >
> > >
> > > --
> > > Alexander Lorenz
> > > http://mapredit.blogspot.com
> > >
> >
> >
> >
> > --
> > Alexander Lorenz
> > http://mapredit.blogspot.com
> >
>
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
Re: jobtracker cannot be started
Posted by "Alexander C.H. Lorenz" <wg...@googlemail.com>.
How large is the blocksize? Hadoop uses a buffer size of 4kb for IO
operations, I can imagine that the balancer take a lot of memory. Check if
they running. Also a good idea will be to increase the space (add more
datanodes). Check the fs.trash.interval setting or not (no settings disable
the trash-facility).
also you can use "# hadoop dfs -expunge"
And, at least, check the limits on the server (limits.conf)
- alex
On Fri, Oct 21, 2011 at 12:09 PM, Peng, Wei <We...@xerox.com> wrote:
> Alex, thank you a lot for helping me. I will figure out how to change
> the conf file. It seems that even chattr -i does not work.
> Just one last question, why restarting the jobtracker needs such a big
> heap size? I had no problem to restart it before the jobtracker hung ?
> One problem of this hadoop cluster that I did not mention is the DFS
> space only has 5% left.
>
> Thanks
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 5:01 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> should, yes ;) I use 2000 in our environment, but depends on the memory
> on
> your servers.
>
> regards,
> Alex
>
> On Fri, Oct 21, 2011 at 10:58 AM, Peng, Wei <We...@xerox.com> wrote:
>
> > Yes, the heap size the default 1000m. /bin/java -Xmx1000m
> > So if I can change the heapsize to be bigger, I should be able to
> solve
> > this problem?
> >
> > Thanks,
> > Wei
> >
> > -----Original Message-----
> > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > Sent: Friday, October 21, 2011 4:53 AM
> > To: common-user@hadoop.apache.org
> > Subject: Re: jobtracker cannot be started
> >
> > looks like that the Heap utilization has exceeded the value set by
> -Xmx.
> > Do
> > a ps waux|grep java @jobtracker |grep -i xmx
> > The heapsize will be set in hadoop-env.sh:
> > export HADOOP_HEAPSIZE=
> >
> > default 1000, I think.
> >
> > - alex
> >
> > On Fri, Oct 21, 2011 at 10:31 AM, Peng, Wei <We...@xerox.com>
> wrote:
> >
> > > Thank you for your quick reply!!
> > >
> > > I cannot change the hadoop conf files because they are owned by a
> > person
> > > who has left the company, though I have the root access. My Java
> > version
> > > is java version "1.5.0_07"
> > > Java(TM) 2 Runtime Environment, Standard Edition (build
> 1.5.0_07-b03)
> > > Java HotSpot(TM) 64-Bit Server VM (build 1.5.0_07-b03, mixed mode)
> > >
> > > The log on http://jobtracker:50030/stacks is
> > > Process Thread Dump:
> > > 26 active threads
> > > Thread 53 (1424598978@qtp0-5):
> > > State: RUNNABLE
> > > Blocked count: 0
> > > Waited count: 29
> > > Stack:
> > > sun.management.ThreadImpl.getThreadInfo0(Native Method)
> > > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
> > > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
> > >
> > >
> >
> org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.j
> > > ava:149)
> > >
> > >
> >
> org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:505
> > > )
> > > javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> > > javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> > >
> > >
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
> > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)
> > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > Thread 43 (Trash Emptier):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 183
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > > org.apache.hadoop.fs.Trash$Emptier.run(Trash.java:234)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 36 (IPC Server handler 9 on 9000):
> > > State: WAITING
> > > Blocked count: 32
> > > Waited count: 3444
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 35 (IPC Server handler 8 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3446
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 34 (IPC Server handler 7 on 9000):
> > > State: WAITING
> > > Blocked count: 30
> > > Waited count: 3451
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 33 (IPC Server handler 6 on 9000):
> > > State: WAITING
> > > Blocked count: 33
> > > Waited count: 3445
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 32 (IPC Server handler 5 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3448
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 31 (IPC Server handler 4 on 9000):
> > > State: WAITING
> > > Blocked count: 33
> > > Waited count: 3446
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 30 (IPC Server handler 3 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3450
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 29 (IPC Server handler 2 on 9000):
> > > State: WAITING
> > > Blocked count: 29
> > > Waited count: 3449
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 28 (IPC Server handler 1 on 9000):
> > > State: WAITING
> > > Blocked count: 36
> > > Waited count: 3445
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 27 (IPC Server handler 0 on 9000):
> > > State: WAITING
> > > Blocked count: 44
> > > Waited count: 3445
> > > Waiting on
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > > 59d87f
> > > Stack:
> > > sun.misc.Unsafe.park(Native Method)
> > > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> > >
> > >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > > ait(AbstractQueuedSynchronizer.java:1925)
> > >
> > >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > > 58)
> > > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > > Thread 12 (IPC Server listener on 9000):
> > > State: RUNNABLE
> > > Blocked count: 5
> > > Waited count: 0
> > > Stack:
> > > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
> > > org.apache.hadoop.ipc.Server$Listener.run(Server.java:318)
> > > Thread 13 (IPC Server Responder):
> > > State: RUNNABLE
> > > Blocked count: 1
> > > Waited count: 0
> > > Stack:
> > > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > > org.apache.hadoop.ipc.Server$Responder.run(Server.java:478)
> > > Thread 26 (Timer-0):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 190
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.util.TimerThread.mainLoop(Timer.java:509)
> > > java.util.TimerThread.run(Timer.java:462)
> > > Thread 25 (1947740113@qtp0-1 - Acceptor0
> > > SelectChannelConnector@penguin33:50070):
> > > State: RUNNABLE
> > > Blocked count: 579
> > > Waited count: 5
> > > Stack:
> > > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > >
> > >
> >
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > va:429)
> > >
> > >
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > >
> > >
> >
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > or.java:124)
> > >
> > >
> >
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > 707)
> > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > > Thread 22
> > >
> >
> (org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@67c7
> > > 980c):
> > > State: TIMED_WAITING
> > > Blocked count: 5
> > > Waited count: 127
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
> org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(D
> > > ecommissionManager.java:65)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 21
> > >
> >
> (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@
> > > 2094257f):
> > > State: TIMED_WAITING
> > > Blocked count: 20
> > > Waited count: 1263
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.r
> > > un(FSNamesystem.java:2304)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 20
> > >
> >
> (org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a51127a):
> > > State: TIMED_WAITING
> > > Blocked count: 21
> > > Waited count: 1875
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
> org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseMan
> > > ager.java:349)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 19
> > >
> >
> (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@61
> > > 578aab):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 13
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run
> > > (FSNamesystem.java:2286)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 18
> > >
> >
> (org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$Pending
> > > ReplicationMonitor@2339e351):
> > > State: TIMED_WAITING
> > > Blocked count: 0
> > > Waited count: 13
> > > Stack:
> > > java.lang.Thread.sleep(Native Method)
> > >
> > >
> >
> org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingR
> > > eplicationMonitor.run(PendingReplicationBlocks.java:186)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 9 (RMI TCP Accept-0):
> > > State: RUNNABLE
> > > Blocked count: 0
> > > Waited count: 0
> > > Stack:
> > > java.net.PlainSocketImpl.socketAccept(Native Method)
> > > java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
> > > java.net.ServerSocket.implAccept(ServerSocket.java:453)
> > > java.net.ServerSocket.accept(ServerSocket.java:421)
> > >
> > >
> >
> sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTrans
> > > port.java:369)
> > >
> > >
> >
> sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
> > > java.lang.Thread.run(Thread.java:619)
> > > Thread 4 (Signal Dispatcher):
> > > State: RUNNABLE
> > > Blocked count: 0
> > > Waited count: 0
> > > Stack:
> > > Thread 3 (Finalizer):
> > > State: WAITING
> > > Blocked count: 0
> > > Waited count: 40
> > > Waiting on java.lang.ref.ReferenceQueue$Lock@22f62eba
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
> > > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
> > > java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
> > > Thread 2 (Reference Handler):
> > > State: WAITING
> > > Blocked count: 1
> > > Waited count: 39
> > > Waiting on java.lang.ref.Reference$Lock@646d6aa0
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.lang.Object.wait(Object.java:485)
> > > java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
> > > Thread 1 (main):
> > > State: WAITING
> > > Blocked count: 10
> > > Waited count: 10
> > > Waiting on org.apache.hadoop.ipc.RPC$Server@41f6321
> > > Stack:
> > > java.lang.Object.wait(Native Method)
> > > java.lang.Object.wait(Object.java:485)
> > > org.apache.hadoop.ipc.Server.join(Server.java:1122)
> > >
> > >
> >
> org.apache.hadoop.hdfs.server.namenode.NameNode.join(NameNode.java:292)
> > >
> > >
> >
> org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:966)
> > >
> > > Wei
> > >
> > > -----Original Message-----
> > > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > > Sent: Friday, October 21, 2011 4:15 AM
> > > To: common-user@hadoop.apache.org
> > > Subject: Re: jobtracker cannot be started
> > >
> > > add into hadoop-env.sh the opts to the jdk-call. The logs should be
> > > accessible at he hadoop-log-directory.
> > >
> > > Also check http://jobtracker:50030/stacks - thats the same as jstack
> > > (jstack
> > > PID). Also you can use jstack -F PID to get a corefile (similar to
> > > /stacks I
> > > think) @jobtracker.
> > >
> > > Are you using 64bit-JDK? Which version?
> > >
> > > regards,
> > > Alex
> > >
> > > On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com>
> > wrote:
> > >
> > > > I am using the default heap size, which is 1000MB. The jobtracker
> > hung
> > > > when only I was running one job. Now I could not even restart the
> > > > jobtracker.
> > > > Can you teach me how to turn on GC logging in hadoop?
> > > >
> > > > Thanks!
> > > > Wei
> > > >
> > > > -----Original Message-----
> > > > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > > > Sent: Friday, October 21, 2011 3:54 AM
> > > > To: common-user@hadoop.apache.org
> > > > Subject: Re: jobtracker cannot be started
> > > >
> > > > Hi,
> > > >
> > > > what are the heap size you given at the jobtracker? And how much
> > jobs
> > > /
> > > > users / tasks are run? What say a log?
> > > > Turn on GC logging:
> > > >
> > http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
> > > >
> > > > - Alex
> > > >
> > > >
> > > > On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com>
> > wrote:
> > > >
> > > > > Hi,
> > > > >
> > > > >
> > > > >
> > > > > When I was running a job on hadoop with 75% mappers finished,
> the
> > > > > jobtracker hung so that I cannot access
> > > > > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung
> > as
> > > > > well.
> > > > >
> > > > >
> > > > >
> > > > > Then I stopped jobtracker and restarted it. However, the
> > jobtracker
> > > > > cannot be started. I received error message from
> > jobtracker.log.out
> > > > > saying
> > > > >
> > > > >
> > > > >
> > > > > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError:
> > Java
> > > > heap
> > > > > space
> > > > >
> > > > > at
> > > > >
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> > > > >
> > > > > at
> > > > >
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> > > > >
> > > > > at
> > > > >
> > org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> > > > >
> > > > > at
> > > org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> > > > >
> > > > > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> > > > >
> > > > > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> > > > >
> > > > > at $Proxy4.renewLease(Unknown Source)
> > > > >
> > > > > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
> > > Source)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > > > > cationHandler.java:82)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > > > > Handler.java:59)
> > > > >
> > > > > at $Proxy4.renewLease(Unknown Source)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> > > > >
> > > > > at
> > > > >
> > >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> > > > >
> > > > > at java.lang.Thread.run(Thread.java:619)
> > > > >
> > > > > Exception in thread "expireTrackers" java.lang.OutOfMemoryError:
> > > Java
> > > > > heap space
> > > > >
> > > > > at java.util.Arrays.copyOf(Arrays.java:2882)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > > > > a:100)
> > > > >
> > > > > at
> > > > >
> > >
> java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> > > > >
> > > > > at java.lang.StringBuffer.append(StringBuffer.java:224)
> > > > >
> > > > > at
> > > org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > > > > a:1354)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > > > > 2)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > > > > va:849)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > > > > 3)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > > > > )
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > > > > 48)
> > > > >
> > > > > at java.lang.Thread.run(Thread.java:619)
> > > > >
> > > > > Exception in thread "IPC Server listener on 9001"
> > > > > java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > > > va:587)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > > > or.java:124)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > > > 707)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > > > > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM
> to
> > > > > handler- the VM may need to be forcibly terminated
> > > > >
> > > > > Exception in thread "retireJobs" java.lang.OutOfMemoryError:
> Java
> > > heap
> > > > > space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > Exception in thread "1586500830@qtp0-37"
> > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > Exception in thread "1513282332@qtp0-29"
> > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > > > > .java:281)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > > > > er.java:44)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > > > > va:213)
> > > > >
> > > > > at java.util.TimerThread.mainLoop(Timer.java:512)
> > > > >
> > > > > at java.util.TimerThread.run(Timer.java:462)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > Exception in thread "IPC Server Responder"
> > > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 429)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > Exception in thread "expireLaunchingTasks"
> > > java.lang.OutOfMemoryError:
> > > > > Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 429)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > > 16)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > > >
> > > > > at
> > > > >
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > > Collection.java:230)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > > >
> > > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > > java.lang.reflect.InvocationTargetException
> > > > >
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > > a:39)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > > Impl.java:25)
> > > > >
> > > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > > >
> > > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > > >
> > > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > > ction.java:864)
> > > > >
> > > > > at
> > > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > > >
> > > > > at
> > > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > > 409)
> > > > >
> > > > > at
> > > > >
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > > :522)
> > > > >
> > > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > > >
> > > > >
> > > > >
> > > > > I have no idea how to solve this problem. The memory on the
> > > jobtracker
> > > > > server seems ok
> > > > >
> > > > > total used free shared buffers
> > > > > cached
> > > > >
> > > > > Mem: 7148044 3578024 3570020 0 673768
> > > > > 467340
> > > > >
> > > > > -/+ buffers/cache: 2436916 4711128
> > > > >
> > > > > Swap: 1020116 160 1019956
> > > > >
> > > > >
> > > > >
> > > > > Can someone help me on this?
> > > > >
> > > > >
> > > > >
> > > > > Thanks a lot
> > > > >
> > > > > Wei
> > > > >
> > > > >
> > > >
> > > >
> > > > --
> > > > Alexander Lorenz
> > > > http://mapredit.blogspot.com
> > > >
> > >
> > >
> > >
> > > --
> > > Alexander Lorenz
> > > http://mapredit.blogspot.com
> > >
> >
> >
> >
> > --
> > Alexander Lorenz
> > http://mapredit.blogspot.com
> >
>
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
RE: jobtracker cannot be started
Posted by "Peng, Wei" <We...@xerox.com>.
Alex, thank you a lot for helping me. I will figure out how to change
the conf file. It seems that even chattr -i does not work.
Just one last question, why restarting the jobtracker needs such a big
heap size? I had no problem to restart it before the jobtracker hung ?
One problem of this hadoop cluster that I did not mention is the DFS
space only has 5% left.
Thanks
Wei
-----Original Message-----
From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
Sent: Friday, October 21, 2011 5:01 AM
To: common-user@hadoop.apache.org
Subject: Re: jobtracker cannot be started
should, yes ;) I use 2000 in our environment, but depends on the memory
on
your servers.
regards,
Alex
On Fri, Oct 21, 2011 at 10:58 AM, Peng, Wei <We...@xerox.com> wrote:
> Yes, the heap size the default 1000m. /bin/java -Xmx1000m
> So if I can change the heapsize to be bigger, I should be able to
solve
> this problem?
>
> Thanks,
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 4:53 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> looks like that the Heap utilization has exceeded the value set by
-Xmx.
> Do
> a ps waux|grep java @jobtracker |grep -i xmx
> The heapsize will be set in hadoop-env.sh:
> export HADOOP_HEAPSIZE=
>
> default 1000, I think.
>
> - alex
>
> On Fri, Oct 21, 2011 at 10:31 AM, Peng, Wei <We...@xerox.com>
wrote:
>
> > Thank you for your quick reply!!
> >
> > I cannot change the hadoop conf files because they are owned by a
> person
> > who has left the company, though I have the root access. My Java
> version
> > is java version "1.5.0_07"
> > Java(TM) 2 Runtime Environment, Standard Edition (build
1.5.0_07-b03)
> > Java HotSpot(TM) 64-Bit Server VM (build 1.5.0_07-b03, mixed mode)
> >
> > The log on http://jobtracker:50030/stacks is
> > Process Thread Dump:
> > 26 active threads
> > Thread 53 (1424598978@qtp0-5):
> > State: RUNNABLE
> > Blocked count: 0
> > Waited count: 29
> > Stack:
> > sun.management.ThreadImpl.getThreadInfo0(Native Method)
> > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
> > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
> >
> >
>
org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.j
> > ava:149)
> >
> >
>
org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:505
> > )
> > javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> > javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> >
> >
org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
> >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)
> >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > org.mortbay.jetty.Server.handle(Server.java:324)
> >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > Thread 43 (Trash Emptier):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 183
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> > org.apache.hadoop.fs.Trash$Emptier.run(Trash.java:234)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 36 (IPC Server handler 9 on 9000):
> > State: WAITING
> > Blocked count: 32
> > Waited count: 3444
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 35 (IPC Server handler 8 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3446
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 34 (IPC Server handler 7 on 9000):
> > State: WAITING
> > Blocked count: 30
> > Waited count: 3451
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 33 (IPC Server handler 6 on 9000):
> > State: WAITING
> > Blocked count: 33
> > Waited count: 3445
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 32 (IPC Server handler 5 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3448
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 31 (IPC Server handler 4 on 9000):
> > State: WAITING
> > Blocked count: 33
> > Waited count: 3446
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 30 (IPC Server handler 3 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3450
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 29 (IPC Server handler 2 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3449
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 28 (IPC Server handler 1 on 9000):
> > State: WAITING
> > Blocked count: 36
> > Waited count: 3445
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 27 (IPC Server handler 0 on 9000):
> > State: WAITING
> > Blocked count: 44
> > Waited count: 3445
> > Waiting on
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 12 (IPC Server listener on 9000):
> > State: RUNNABLE
> > Blocked count: 5
> > Waited count: 0
> > Stack:
> > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
> > org.apache.hadoop.ipc.Server$Listener.run(Server.java:318)
> > Thread 13 (IPC Server Responder):
> > State: RUNNABLE
> > Blocked count: 1
> > Waited count: 0
> > Stack:
> > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > org.apache.hadoop.ipc.Server$Responder.run(Server.java:478)
> > Thread 26 (Timer-0):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 190
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.util.TimerThread.mainLoop(Timer.java:509)
> > java.util.TimerThread.run(Timer.java:462)
> > Thread 25 (1947740113@qtp0-1 - Acceptor0
> > SelectChannelConnector@penguin33:50070):
> > State: RUNNABLE
> > Blocked count: 579
> > Waited count: 5
> > Stack:
> > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> >
> >
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > va:429)
> >
> >
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> >
> >
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > or.java:124)
> >
> >
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > 707)
> >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> > Thread 22
> >
>
(org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@67c7
> > 980c):
> > State: TIMED_WAITING
> > Blocked count: 5
> > Waited count: 127
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
>
org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(D
> > ecommissionManager.java:65)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 21
> >
>
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@
> > 2094257f):
> > State: TIMED_WAITING
> > Blocked count: 20
> > Waited count: 1263
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
>
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.r
> > un(FSNamesystem.java:2304)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 20
> >
>
(org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a51127a):
> > State: TIMED_WAITING
> > Blocked count: 21
> > Waited count: 1875
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
>
org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseMan
> > ager.java:349)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 19
> >
>
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@61
> > 578aab):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 13
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
>
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run
> > (FSNamesystem.java:2286)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 18
> >
>
(org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$Pending
> > ReplicationMonitor@2339e351):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 13
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
>
org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingR
> > eplicationMonitor.run(PendingReplicationBlocks.java:186)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 9 (RMI TCP Accept-0):
> > State: RUNNABLE
> > Blocked count: 0
> > Waited count: 0
> > Stack:
> > java.net.PlainSocketImpl.socketAccept(Native Method)
> > java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
> > java.net.ServerSocket.implAccept(ServerSocket.java:453)
> > java.net.ServerSocket.accept(ServerSocket.java:421)
> >
> >
>
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTrans
> > port.java:369)
> >
> >
>
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 4 (Signal Dispatcher):
> > State: RUNNABLE
> > Blocked count: 0
> > Waited count: 0
> > Stack:
> > Thread 3 (Finalizer):
> > State: WAITING
> > Blocked count: 0
> > Waited count: 40
> > Waiting on java.lang.ref.ReferenceQueue$Lock@22f62eba
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
> > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
> > java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
> > Thread 2 (Reference Handler):
> > State: WAITING
> > Blocked count: 1
> > Waited count: 39
> > Waiting on java.lang.ref.Reference$Lock@646d6aa0
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.lang.Object.wait(Object.java:485)
> > java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
> > Thread 1 (main):
> > State: WAITING
> > Blocked count: 10
> > Waited count: 10
> > Waiting on org.apache.hadoop.ipc.RPC$Server@41f6321
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.lang.Object.wait(Object.java:485)
> > org.apache.hadoop.ipc.Server.join(Server.java:1122)
> >
> >
>
org.apache.hadoop.hdfs.server.namenode.NameNode.join(NameNode.java:292)
> >
> >
>
org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:966)
> >
> > Wei
> >
> > -----Original Message-----
> > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > Sent: Friday, October 21, 2011 4:15 AM
> > To: common-user@hadoop.apache.org
> > Subject: Re: jobtracker cannot be started
> >
> > add into hadoop-env.sh the opts to the jdk-call. The logs should be
> > accessible at he hadoop-log-directory.
> >
> > Also check http://jobtracker:50030/stacks - thats the same as jstack
> > (jstack
> > PID). Also you can use jstack -F PID to get a corefile (similar to
> > /stacks I
> > think) @jobtracker.
> >
> > Are you using 64bit-JDK? Which version?
> >
> > regards,
> > Alex
> >
> > On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com>
> wrote:
> >
> > > I am using the default heap size, which is 1000MB. The jobtracker
> hung
> > > when only I was running one job. Now I could not even restart the
> > > jobtracker.
> > > Can you teach me how to turn on GC logging in hadoop?
> > >
> > > Thanks!
> > > Wei
> > >
> > > -----Original Message-----
> > > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > > Sent: Friday, October 21, 2011 3:54 AM
> > > To: common-user@hadoop.apache.org
> > > Subject: Re: jobtracker cannot be started
> > >
> > > Hi,
> > >
> > > what are the heap size you given at the jobtracker? And how much
> jobs
> > /
> > > users / tasks are run? What say a log?
> > > Turn on GC logging:
> > >
> http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
> > >
> > > - Alex
> > >
> > >
> > > On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com>
> wrote:
> > >
> > > > Hi,
> > > >
> > > >
> > > >
> > > > When I was running a job on hadoop with 75% mappers finished,
the
> > > > jobtracker hung so that I cannot access
> > > > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung
> as
> > > > well.
> > > >
> > > >
> > > >
> > > > Then I stopped jobtracker and restarted it. However, the
> jobtracker
> > > > cannot be started. I received error message from
> jobtracker.log.out
> > > > saying
> > > >
> > > >
> > > >
> > > > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError:
> Java
> > > heap
> > > > space
> > > >
> > > > at
> > > >
java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> > > >
> > > > at
> > > >
java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> > > >
> > > > at
> > > >
> org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> > > >
> > > > at
> > org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> > > >
> > > > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> > > >
> > > > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> > > >
> > > > at $Proxy4.renewLease(Unknown Source)
> > > >
> > > > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
> > Source)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > > > cationHandler.java:82)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > > > Handler.java:59)
> > > >
> > > > at $Proxy4.renewLease(Unknown Source)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> > > >
> > > > at
> > > >
> >
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> > > >
> > > > at java.lang.Thread.run(Thread.java:619)
> > > >
> > > > Exception in thread "expireTrackers" java.lang.OutOfMemoryError:
> > Java
> > > > heap space
> > > >
> > > > at java.util.Arrays.copyOf(Arrays.java:2882)
> > > >
> > > > at
> > > >
> > >
> >
>
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > > > a:100)
> > > >
> > > > at
> > > >
> >
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> > > >
> > > > at java.lang.StringBuffer.append(StringBuffer.java:224)
> > > >
> > > > at
> > org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > > > a:1354)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > > > 2)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > > > va:849)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > > > 3)
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > > > )
> > > >
> > > > at
> > > >
> > >
> >
>
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > > > 48)
> > > >
> > > > at java.lang.Thread.run(Thread.java:619)
> > > >
> > > > Exception in thread "IPC Server listener on 9001"
> > > > java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > > va:587)
> > > >
> > > > at
> > > >
> >
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > > or.java:124)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > > 707)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > > > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM
to
> > > > handler- the VM may need to be forcibly terminated
> > > >
> > > > Exception in thread "retireJobs" java.lang.OutOfMemoryError:
Java
> > heap
> > > > space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > Exception in thread "1586500830@qtp0-37"
> java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > Exception in thread "1513282332@qtp0-29"
> java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > > > .java:281)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > > > er.java:44)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > > > va:213)
> > > >
> > > > at java.util.TimerThread.mainLoop(Timer.java:512)
> > > >
> > > > at java.util.TimerThread.run(Timer.java:462)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > Exception in thread "IPC Server Responder"
> > java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 429)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > Exception in thread "expireLaunchingTasks"
> > java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 429)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > >
> > > >
> > > > I have no idea how to solve this problem. The memory on the
> > jobtracker
> > > > server seems ok
> > > >
> > > > total used free shared buffers
> > > > cached
> > > >
> > > > Mem: 7148044 3578024 3570020 0 673768
> > > > 467340
> > > >
> > > > -/+ buffers/cache: 2436916 4711128
> > > >
> > > > Swap: 1020116 160 1019956
> > > >
> > > >
> > > >
> > > > Can someone help me on this?
> > > >
> > > >
> > > >
> > > > Thanks a lot
> > > >
> > > > Wei
> > > >
> > > >
> > >
> > >
> > > --
> > > Alexander Lorenz
> > > http://mapredit.blogspot.com
> > >
> >
> >
> >
> > --
> > Alexander Lorenz
> > http://mapredit.blogspot.com
> >
>
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
Re: jobtracker cannot be started
Posted by "Alexander C.H. Lorenz" <wg...@googlemail.com>.
should, yes ;) I use 2000 in our environment, but depends on the memory on
your servers.
regards,
Alex
On Fri, Oct 21, 2011 at 10:58 AM, Peng, Wei <We...@xerox.com> wrote:
> Yes, the heap size the default 1000m. /bin/java -Xmx1000m
> So if I can change the heapsize to be bigger, I should be able to solve
> this problem?
>
> Thanks,
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 4:53 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> looks like that the Heap utilization has exceeded the value set by -Xmx.
> Do
> a ps waux|grep java @jobtracker |grep -i xmx
> The heapsize will be set in hadoop-env.sh:
> export HADOOP_HEAPSIZE=
>
> default 1000, I think.
>
> - alex
>
> On Fri, Oct 21, 2011 at 10:31 AM, Peng, Wei <We...@xerox.com> wrote:
>
> > Thank you for your quick reply!!
> >
> > I cannot change the hadoop conf files because they are owned by a
> person
> > who has left the company, though I have the root access. My Java
> version
> > is java version "1.5.0_07"
> > Java(TM) 2 Runtime Environment, Standard Edition (build 1.5.0_07-b03)
> > Java HotSpot(TM) 64-Bit Server VM (build 1.5.0_07-b03, mixed mode)
> >
> > The log on http://jobtracker:50030/stacks is
> > Process Thread Dump:
> > 26 active threads
> > Thread 53 (1424598978@qtp0-5):
> > State: RUNNABLE
> > Blocked count: 0
> > Waited count: 29
> > Stack:
> > sun.management.ThreadImpl.getThreadInfo0(Native Method)
> > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
> > sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
> >
> >
> org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.j
> > ava:149)
> >
> >
> org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:505
> > )
> > javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> > javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
> >
> > org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
> >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)
> >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > org.mortbay.jetty.Server.handle(Server.java:324)
> >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > Thread 43 (Trash Emptier):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 183
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> > org.apache.hadoop.fs.Trash$Emptier.run(Trash.java:234)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 36 (IPC Server handler 9 on 9000):
> > State: WAITING
> > Blocked count: 32
> > Waited count: 3444
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 35 (IPC Server handler 8 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3446
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 34 (IPC Server handler 7 on 9000):
> > State: WAITING
> > Blocked count: 30
> > Waited count: 3451
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 33 (IPC Server handler 6 on 9000):
> > State: WAITING
> > Blocked count: 33
> > Waited count: 3445
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 32 (IPC Server handler 5 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3448
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 31 (IPC Server handler 4 on 9000):
> > State: WAITING
> > Blocked count: 33
> > Waited count: 3446
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 30 (IPC Server handler 3 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3450
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 29 (IPC Server handler 2 on 9000):
> > State: WAITING
> > Blocked count: 29
> > Waited count: 3449
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 28 (IPC Server handler 1 on 9000):
> > State: WAITING
> > Blocked count: 36
> > Waited count: 3445
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 27 (IPC Server handler 0 on 9000):
> > State: WAITING
> > Blocked count: 44
> > Waited count: 3445
> > Waiting on
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> > 59d87f
> > Stack:
> > sun.misc.Unsafe.park(Native Method)
> > java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
> >
> >
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> > ait(AbstractQueuedSynchronizer.java:1925)
> >
> >
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> > 58)
> > org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> > Thread 12 (IPC Server listener on 9000):
> > State: RUNNABLE
> > Blocked count: 5
> > Waited count: 0
> > Stack:
> > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
> > org.apache.hadoop.ipc.Server$Listener.run(Server.java:318)
> > Thread 13 (IPC Server Responder):
> > State: RUNNABLE
> > Blocked count: 1
> > Waited count: 0
> > Stack:
> > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> > org.apache.hadoop.ipc.Server$Responder.run(Server.java:478)
> > Thread 26 (Timer-0):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 190
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.util.TimerThread.mainLoop(Timer.java:509)
> > java.util.TimerThread.run(Timer.java:462)
> > Thread 25 (1947740113@qtp0-1 - Acceptor0
> > SelectChannelConnector@penguin33:50070):
> > State: RUNNABLE
> > Blocked count: 579
> > Waited count: 5
> > Stack:
> > sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> > sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> > sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> > sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> > sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> >
> >
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > va:429)
> >
> > org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> >
> >
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > or.java:124)
> >
> >
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > 707)
> >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> > Thread 22
> >
> (org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@67c7
> > 980c):
> > State: TIMED_WAITING
> > Blocked count: 5
> > Waited count: 127
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
> org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(D
> > ecommissionManager.java:65)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 21
> >
> (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@
> > 2094257f):
> > State: TIMED_WAITING
> > Blocked count: 20
> > Waited count: 1263
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.r
> > un(FSNamesystem.java:2304)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 20
> >
> (org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a51127a):
> > State: TIMED_WAITING
> > Blocked count: 21
> > Waited count: 1875
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
> org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseMan
> > ager.java:349)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 19
> >
> (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@61
> > 578aab):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 13
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run
> > (FSNamesystem.java:2286)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 18
> >
> (org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$Pending
> > ReplicationMonitor@2339e351):
> > State: TIMED_WAITING
> > Blocked count: 0
> > Waited count: 13
> > Stack:
> > java.lang.Thread.sleep(Native Method)
> >
> >
> org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingR
> > eplicationMonitor.run(PendingReplicationBlocks.java:186)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 9 (RMI TCP Accept-0):
> > State: RUNNABLE
> > Blocked count: 0
> > Waited count: 0
> > Stack:
> > java.net.PlainSocketImpl.socketAccept(Native Method)
> > java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
> > java.net.ServerSocket.implAccept(ServerSocket.java:453)
> > java.net.ServerSocket.accept(ServerSocket.java:421)
> >
> >
> sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTrans
> > port.java:369)
> >
> >
> sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
> > java.lang.Thread.run(Thread.java:619)
> > Thread 4 (Signal Dispatcher):
> > State: RUNNABLE
> > Blocked count: 0
> > Waited count: 0
> > Stack:
> > Thread 3 (Finalizer):
> > State: WAITING
> > Blocked count: 0
> > Waited count: 40
> > Waiting on java.lang.ref.ReferenceQueue$Lock@22f62eba
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
> > java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
> > java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
> > Thread 2 (Reference Handler):
> > State: WAITING
> > Blocked count: 1
> > Waited count: 39
> > Waiting on java.lang.ref.Reference$Lock@646d6aa0
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.lang.Object.wait(Object.java:485)
> > java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
> > Thread 1 (main):
> > State: WAITING
> > Blocked count: 10
> > Waited count: 10
> > Waiting on org.apache.hadoop.ipc.RPC$Server@41f6321
> > Stack:
> > java.lang.Object.wait(Native Method)
> > java.lang.Object.wait(Object.java:485)
> > org.apache.hadoop.ipc.Server.join(Server.java:1122)
> >
> >
> org.apache.hadoop.hdfs.server.namenode.NameNode.join(NameNode.java:292)
> >
> >
> org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:966)
> >
> > Wei
> >
> > -----Original Message-----
> > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > Sent: Friday, October 21, 2011 4:15 AM
> > To: common-user@hadoop.apache.org
> > Subject: Re: jobtracker cannot be started
> >
> > add into hadoop-env.sh the opts to the jdk-call. The logs should be
> > accessible at he hadoop-log-directory.
> >
> > Also check http://jobtracker:50030/stacks - thats the same as jstack
> > (jstack
> > PID). Also you can use jstack -F PID to get a corefile (similar to
> > /stacks I
> > think) @jobtracker.
> >
> > Are you using 64bit-JDK? Which version?
> >
> > regards,
> > Alex
> >
> > On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com>
> wrote:
> >
> > > I am using the default heap size, which is 1000MB. The jobtracker
> hung
> > > when only I was running one job. Now I could not even restart the
> > > jobtracker.
> > > Can you teach me how to turn on GC logging in hadoop?
> > >
> > > Thanks!
> > > Wei
> > >
> > > -----Original Message-----
> > > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > > Sent: Friday, October 21, 2011 3:54 AM
> > > To: common-user@hadoop.apache.org
> > > Subject: Re: jobtracker cannot be started
> > >
> > > Hi,
> > >
> > > what are the heap size you given at the jobtracker? And how much
> jobs
> > /
> > > users / tasks are run? What say a log?
> > > Turn on GC logging:
> > >
> http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
> > >
> > > - Alex
> > >
> > >
> > > On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com>
> wrote:
> > >
> > > > Hi,
> > > >
> > > >
> > > >
> > > > When I was running a job on hadoop with 75% mappers finished, the
> > > > jobtracker hung so that I cannot access
> > > > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung
> as
> > > > well.
> > > >
> > > >
> > > >
> > > > Then I stopped jobtracker and restarted it. However, the
> jobtracker
> > > > cannot be started. I received error message from
> jobtracker.log.out
> > > > saying
> > > >
> > > >
> > > >
> > > > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError:
> Java
> > > heap
> > > > space
> > > >
> > > > at
> > > > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> > > >
> > > > at
> > > > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> > > >
> > > > at
> > > >
> org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> > > >
> > > > at
> > org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> > > >
> > > > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> > > >
> > > > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> > > >
> > > > at $Proxy4.renewLease(Unknown Source)
> > > >
> > > > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
> > Source)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > > > cationHandler.java:82)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > > > Handler.java:59)
> > > >
> > > > at $Proxy4.renewLease(Unknown Source)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> > > >
> > > > at
> > > >
> > org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> > > >
> > > > at java.lang.Thread.run(Thread.java:619)
> > > >
> > > > Exception in thread "expireTrackers" java.lang.OutOfMemoryError:
> > Java
> > > > heap space
> > > >
> > > > at java.util.Arrays.copyOf(Arrays.java:2882)
> > > >
> > > > at
> > > >
> > >
> >
> java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > > > a:100)
> > > >
> > > > at
> > > >
> > java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> > > >
> > > > at java.lang.StringBuffer.append(StringBuffer.java:224)
> > > >
> > > > at
> > org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > > > a:1354)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > > > 2)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > > > va:849)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > > > 3)
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > > > )
> > > >
> > > > at
> > > >
> > >
> >
> org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > > > 48)
> > > >
> > > > at java.lang.Thread.run(Thread.java:619)
> > > >
> > > > Exception in thread "IPC Server listener on 9001"
> > > > java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > > va:587)
> > > >
> > > > at
> > > >
> > org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > > or.java:124)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > > 707)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > > > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> > > > handler- the VM may need to be forcibly terminated
> > > >
> > > > Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java
> > heap
> > > > space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > Exception in thread "1586500830@qtp0-37"
> java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > Exception in thread "1513282332@qtp0-29"
> java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > > > .java:281)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > > > er.java:44)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > > > va:213)
> > > >
> > > > at java.util.TimerThread.mainLoop(Timer.java:512)
> > > >
> > > > at java.util.TimerThread.run(Timer.java:462)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > Exception in thread "IPC Server Responder"
> > java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 429)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > Exception in thread "expireLaunchingTasks"
> > java.lang.OutOfMemoryError:
> > > > Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 429)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > > 16)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > > >
> > > > at
> > > >
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > > Collection.java:230)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > > >
> > > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > > java.lang.reflect.InvocationTargetException
> > > >
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > > a:39)
> > > >
> > > > at
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > > Impl.java:25)
> > > >
> > > > at java.lang.reflect.Method.invoke(Method.java:597)
> > > >
> > > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > > >
> > > > at org.mortbay.log.Log.warn(Log.java:189)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > > ction.java:864)
> > > >
> > > > at
> > org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > > >
> > > > at
> > > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > > 409)
> > > >
> > > > at
> > > >
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > > :522)
> > > >
> > > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > > >
> > > >
> > > >
> > > > I have no idea how to solve this problem. The memory on the
> > jobtracker
> > > > server seems ok
> > > >
> > > > total used free shared buffers
> > > > cached
> > > >
> > > > Mem: 7148044 3578024 3570020 0 673768
> > > > 467340
> > > >
> > > > -/+ buffers/cache: 2436916 4711128
> > > >
> > > > Swap: 1020116 160 1019956
> > > >
> > > >
> > > >
> > > > Can someone help me on this?
> > > >
> > > >
> > > >
> > > > Thanks a lot
> > > >
> > > > Wei
> > > >
> > > >
> > >
> > >
> > > --
> > > Alexander Lorenz
> > > http://mapredit.blogspot.com
> > >
> >
> >
> >
> > --
> > Alexander Lorenz
> > http://mapredit.blogspot.com
> >
>
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
RE: jobtracker cannot be started
Posted by "Peng, Wei" <We...@xerox.com>.
Yes, the heap size the default 1000m. /bin/java -Xmx1000m
So if I can change the heapsize to be bigger, I should be able to solve
this problem?
Thanks,
Wei
-----Original Message-----
From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
Sent: Friday, October 21, 2011 4:53 AM
To: common-user@hadoop.apache.org
Subject: Re: jobtracker cannot be started
looks like that the Heap utilization has exceeded the value set by -Xmx.
Do
a ps waux|grep java @jobtracker |grep -i xmx
The heapsize will be set in hadoop-env.sh:
export HADOOP_HEAPSIZE=
default 1000, I think.
- alex
On Fri, Oct 21, 2011 at 10:31 AM, Peng, Wei <We...@xerox.com> wrote:
> Thank you for your quick reply!!
>
> I cannot change the hadoop conf files because they are owned by a
person
> who has left the company, though I have the root access. My Java
version
> is java version "1.5.0_07"
> Java(TM) 2 Runtime Environment, Standard Edition (build 1.5.0_07-b03)
> Java HotSpot(TM) 64-Bit Server VM (build 1.5.0_07-b03, mixed mode)
>
> The log on http://jobtracker:50030/stacks is
> Process Thread Dump:
> 26 active threads
> Thread 53 (1424598978@qtp0-5):
> State: RUNNABLE
> Blocked count: 0
> Waited count: 29
> Stack:
> sun.management.ThreadImpl.getThreadInfo0(Native Method)
> sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
> sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
>
>
org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.j
> ava:149)
>
>
org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:505
> )
> javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
>
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)
>
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> org.mortbay.jetty.Server.handle(Server.java:324)
>
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> Thread 43 (Trash Emptier):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 183
> Stack:
> java.lang.Thread.sleep(Native Method)
> org.apache.hadoop.fs.Trash$Emptier.run(Trash.java:234)
> java.lang.Thread.run(Thread.java:619)
> Thread 36 (IPC Server handler 9 on 9000):
> State: WAITING
> Blocked count: 32
> Waited count: 3444
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 35 (IPC Server handler 8 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3446
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 34 (IPC Server handler 7 on 9000):
> State: WAITING
> Blocked count: 30
> Waited count: 3451
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 33 (IPC Server handler 6 on 9000):
> State: WAITING
> Blocked count: 33
> Waited count: 3445
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 32 (IPC Server handler 5 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3448
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 31 (IPC Server handler 4 on 9000):
> State: WAITING
> Blocked count: 33
> Waited count: 3446
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 30 (IPC Server handler 3 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3450
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 29 (IPC Server handler 2 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3449
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 28 (IPC Server handler 1 on 9000):
> State: WAITING
> Blocked count: 36
> Waited count: 3445
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 27 (IPC Server handler 0 on 9000):
> State: WAITING
> Blocked count: 44
> Waited count: 3445
> Waiting on
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
>
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
>
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 12 (IPC Server listener on 9000):
> State: RUNNABLE
> Blocked count: 5
> Waited count: 0
> Stack:
> sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
> org.apache.hadoop.ipc.Server$Listener.run(Server.java:318)
> Thread 13 (IPC Server Responder):
> State: RUNNABLE
> Blocked count: 1
> Waited count: 0
> Stack:
> sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> org.apache.hadoop.ipc.Server$Responder.run(Server.java:478)
> Thread 26 (Timer-0):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 190
> Stack:
> java.lang.Object.wait(Native Method)
> java.util.TimerThread.mainLoop(Timer.java:509)
> java.util.TimerThread.run(Timer.java:462)
> Thread 25 (1947740113@qtp0-1 - Acceptor0
> SelectChannelConnector@penguin33:50070):
> State: RUNNABLE
> Blocked count: 579
> Waited count: 5
> Stack:
> sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
>
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> va:429)
>
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
>
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> or.java:124)
>
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> 707)
>
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
> Thread 22
>
(org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@67c7
> 980c):
> State: TIMED_WAITING
> Blocked count: 5
> Waited count: 127
> Stack:
> java.lang.Thread.sleep(Native Method)
>
>
org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(D
> ecommissionManager.java:65)
> java.lang.Thread.run(Thread.java:619)
> Thread 21
>
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@
> 2094257f):
> State: TIMED_WAITING
> Blocked count: 20
> Waited count: 1263
> Stack:
> java.lang.Thread.sleep(Native Method)
>
>
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.r
> un(FSNamesystem.java:2304)
> java.lang.Thread.run(Thread.java:619)
> Thread 20
>
(org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a51127a):
> State: TIMED_WAITING
> Blocked count: 21
> Waited count: 1875
> Stack:
> java.lang.Thread.sleep(Native Method)
>
>
org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseMan
> ager.java:349)
> java.lang.Thread.run(Thread.java:619)
> Thread 19
>
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@61
> 578aab):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 13
> Stack:
> java.lang.Thread.sleep(Native Method)
>
>
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run
> (FSNamesystem.java:2286)
> java.lang.Thread.run(Thread.java:619)
> Thread 18
>
(org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$Pending
> ReplicationMonitor@2339e351):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 13
> Stack:
> java.lang.Thread.sleep(Native Method)
>
>
org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingR
> eplicationMonitor.run(PendingReplicationBlocks.java:186)
> java.lang.Thread.run(Thread.java:619)
> Thread 9 (RMI TCP Accept-0):
> State: RUNNABLE
> Blocked count: 0
> Waited count: 0
> Stack:
> java.net.PlainSocketImpl.socketAccept(Native Method)
> java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
> java.net.ServerSocket.implAccept(ServerSocket.java:453)
> java.net.ServerSocket.accept(ServerSocket.java:421)
>
>
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTrans
> port.java:369)
>
>
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
> java.lang.Thread.run(Thread.java:619)
> Thread 4 (Signal Dispatcher):
> State: RUNNABLE
> Blocked count: 0
> Waited count: 0
> Stack:
> Thread 3 (Finalizer):
> State: WAITING
> Blocked count: 0
> Waited count: 40
> Waiting on java.lang.ref.ReferenceQueue$Lock@22f62eba
> Stack:
> java.lang.Object.wait(Native Method)
> java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
> java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
> java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
> Thread 2 (Reference Handler):
> State: WAITING
> Blocked count: 1
> Waited count: 39
> Waiting on java.lang.ref.Reference$Lock@646d6aa0
> Stack:
> java.lang.Object.wait(Native Method)
> java.lang.Object.wait(Object.java:485)
> java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
> Thread 1 (main):
> State: WAITING
> Blocked count: 10
> Waited count: 10
> Waiting on org.apache.hadoop.ipc.RPC$Server@41f6321
> Stack:
> java.lang.Object.wait(Native Method)
> java.lang.Object.wait(Object.java:485)
> org.apache.hadoop.ipc.Server.join(Server.java:1122)
>
>
org.apache.hadoop.hdfs.server.namenode.NameNode.join(NameNode.java:292)
>
>
org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:966)
>
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 4:15 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> add into hadoop-env.sh the opts to the jdk-call. The logs should be
> accessible at he hadoop-log-directory.
>
> Also check http://jobtracker:50030/stacks - thats the same as jstack
> (jstack
> PID). Also you can use jstack -F PID to get a corefile (similar to
> /stacks I
> think) @jobtracker.
>
> Are you using 64bit-JDK? Which version?
>
> regards,
> Alex
>
> On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com>
wrote:
>
> > I am using the default heap size, which is 1000MB. The jobtracker
hung
> > when only I was running one job. Now I could not even restart the
> > jobtracker.
> > Can you teach me how to turn on GC logging in hadoop?
> >
> > Thanks!
> > Wei
> >
> > -----Original Message-----
> > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > Sent: Friday, October 21, 2011 3:54 AM
> > To: common-user@hadoop.apache.org
> > Subject: Re: jobtracker cannot be started
> >
> > Hi,
> >
> > what are the heap size you given at the jobtracker? And how much
jobs
> /
> > users / tasks are run? What say a log?
> > Turn on GC logging:
> >
http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
> >
> > - Alex
> >
> >
> > On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com>
wrote:
> >
> > > Hi,
> > >
> > >
> > >
> > > When I was running a job on hadoop with 75% mappers finished, the
> > > jobtracker hung so that I cannot access
> > > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung
as
> > > well.
> > >
> > >
> > >
> > > Then I stopped jobtracker and restarted it. However, the
jobtracker
> > > cannot be started. I received error message from
jobtracker.log.out
> > > saying
> > >
> > >
> > >
> > > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError:
Java
> > heap
> > > space
> > >
> > > at
> > > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> > >
> > > at
> > > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> > >
> > > at
> > >
org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> > >
> > > at
> org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> > >
> > > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> > >
> > > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> > >
> > > at $Proxy4.renewLease(Unknown Source)
> > >
> > > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
> Source)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > > cationHandler.java:82)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > > Handler.java:59)
> > >
> > > at $Proxy4.renewLease(Unknown Source)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> > >
> > > at
> > >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> > >
> > > at java.lang.Thread.run(Thread.java:619)
> > >
> > > Exception in thread "expireTrackers" java.lang.OutOfMemoryError:
> Java
> > > heap space
> > >
> > > at java.util.Arrays.copyOf(Arrays.java:2882)
> > >
> > > at
> > >
> >
>
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > > a:100)
> > >
> > > at
> > >
> java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> > >
> > > at java.lang.StringBuffer.append(StringBuffer.java:224)
> > >
> > > at
> org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > > a:1354)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > > 2)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > > va:849)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > > 3)
> > >
> > > at
> > >
> >
>
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > > )
> > >
> > > at
> > >
> >
>
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > > 48)
> > >
> > > at java.lang.Thread.run(Thread.java:619)
> > >
> > > Exception in thread "IPC Server listener on 9001"
> > > java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > va:587)
> > >
> > > at
> > >
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > or.java:124)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > 707)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:189)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> > > handler- the VM may need to be forcibly terminated
> > >
> > > Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java
> heap
> > > space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > Exception in thread "1586500830@qtp0-37"
java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > Exception in thread "1513282332@qtp0-29"
java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > > .java:281)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > > er.java:44)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > > va:213)
> > >
> > > at java.util.TimerThread.mainLoop(Timer.java:512)
> > >
> > > at java.util.TimerThread.run(Timer.java:462)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > Exception in thread "IPC Server Responder"
> java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 429)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > Exception in thread "expireLaunchingTasks"
> java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:189)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 429)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:189)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > >
> > > at
> > >
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > >
> > >
> > > I have no idea how to solve this problem. The memory on the
> jobtracker
> > > server seems ok
> > >
> > > total used free shared buffers
> > > cached
> > >
> > > Mem: 7148044 3578024 3570020 0 673768
> > > 467340
> > >
> > > -/+ buffers/cache: 2436916 4711128
> > >
> > > Swap: 1020116 160 1019956
> > >
> > >
> > >
> > > Can someone help me on this?
> > >
> > >
> > >
> > > Thanks a lot
> > >
> > > Wei
> > >
> > >
> >
> >
> > --
> > Alexander Lorenz
> > http://mapredit.blogspot.com
> >
>
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
Re: jobtracker cannot be started
Posted by "Alexander C.H. Lorenz" <wg...@googlemail.com>.
looks like that the Heap utilization has exceeded the value set by -Xmx. Do
a ps waux|grep java @jobtracker |grep -i xmx
The heapsize will be set in hadoop-env.sh:
export HADOOP_HEAPSIZE=
default 1000, I think.
- alex
On Fri, Oct 21, 2011 at 10:31 AM, Peng, Wei <We...@xerox.com> wrote:
> Thank you for your quick reply!!
>
> I cannot change the hadoop conf files because they are owned by a person
> who has left the company, though I have the root access. My Java version
> is java version "1.5.0_07"
> Java(TM) 2 Runtime Environment, Standard Edition (build 1.5.0_07-b03)
> Java HotSpot(TM) 64-Bit Server VM (build 1.5.0_07-b03, mixed mode)
>
> The log on http://jobtracker:50030/stacks is
> Process Thread Dump:
> 26 active threads
> Thread 53 (1424598978@qtp0-5):
> State: RUNNABLE
> Blocked count: 0
> Waited count: 29
> Stack:
> sun.management.ThreadImpl.getThreadInfo0(Native Method)
> sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
> sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
>
> org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.j
> ava:149)
>
> org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:505
> )
> javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
> javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
>
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
>
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)
>
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> org.mortbay.jetty.Server.handle(Server.java:324)
>
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> Thread 43 (Trash Emptier):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 183
> Stack:
> java.lang.Thread.sleep(Native Method)
> org.apache.hadoop.fs.Trash$Emptier.run(Trash.java:234)
> java.lang.Thread.run(Thread.java:619)
> Thread 36 (IPC Server handler 9 on 9000):
> State: WAITING
> Blocked count: 32
> Waited count: 3444
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 35 (IPC Server handler 8 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3446
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 34 (IPC Server handler 7 on 9000):
> State: WAITING
> Blocked count: 30
> Waited count: 3451
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 33 (IPC Server handler 6 on 9000):
> State: WAITING
> Blocked count: 33
> Waited count: 3445
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 32 (IPC Server handler 5 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3448
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 31 (IPC Server handler 4 on 9000):
> State: WAITING
> Blocked count: 33
> Waited count: 3446
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 30 (IPC Server handler 3 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3450
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 29 (IPC Server handler 2 on 9000):
> State: WAITING
> Blocked count: 29
> Waited count: 3449
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 28 (IPC Server handler 1 on 9000):
> State: WAITING
> Blocked count: 36
> Waited count: 3445
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 27 (IPC Server handler 0 on 9000):
> State: WAITING
> Blocked count: 44
> Waited count: 3445
> Waiting on
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
> 59d87f
> Stack:
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
> ait(AbstractQueuedSynchronizer.java:1925)
>
> java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
> 58)
> org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
> Thread 12 (IPC Server listener on 9000):
> State: RUNNABLE
> Blocked count: 5
> Waited count: 0
> Stack:
> sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
> org.apache.hadoop.ipc.Server$Listener.run(Server.java:318)
> Thread 13 (IPC Server Responder):
> State: RUNNABLE
> Blocked count: 1
> Waited count: 0
> Stack:
> sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
> org.apache.hadoop.ipc.Server$Responder.run(Server.java:478)
> Thread 26 (Timer-0):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 190
> Stack:
> java.lang.Object.wait(Native Method)
> java.util.TimerThread.mainLoop(Timer.java:509)
> java.util.TimerThread.run(Timer.java:462)
> Thread 25 (1947740113@qtp0-1 - Acceptor0
> SelectChannelConnector@penguin33:50070):
> State: RUNNABLE
> Blocked count: 579
> Waited count: 5
> Stack:
> sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
> sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
> sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
> sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
> sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
>
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> va:429)
>
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
>
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> or.java:124)
>
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> 707)
>
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
> Thread 22
> (org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@67c7
> 980c):
> State: TIMED_WAITING
> Blocked count: 5
> Waited count: 127
> Stack:
> java.lang.Thread.sleep(Native Method)
>
> org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(D
> ecommissionManager.java:65)
> java.lang.Thread.run(Thread.java:619)
> Thread 21
> (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@
> 2094257f):
> State: TIMED_WAITING
> Blocked count: 20
> Waited count: 1263
> Stack:
> java.lang.Thread.sleep(Native Method)
>
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.r
> un(FSNamesystem.java:2304)
> java.lang.Thread.run(Thread.java:619)
> Thread 20
> (org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a51127a):
> State: TIMED_WAITING
> Blocked count: 21
> Waited count: 1875
> Stack:
> java.lang.Thread.sleep(Native Method)
>
> org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseMan
> ager.java:349)
> java.lang.Thread.run(Thread.java:619)
> Thread 19
> (org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@61
> 578aab):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 13
> Stack:
> java.lang.Thread.sleep(Native Method)
>
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run
> (FSNamesystem.java:2286)
> java.lang.Thread.run(Thread.java:619)
> Thread 18
> (org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$Pending
> ReplicationMonitor@2339e351):
> State: TIMED_WAITING
> Blocked count: 0
> Waited count: 13
> Stack:
> java.lang.Thread.sleep(Native Method)
>
> org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingR
> eplicationMonitor.run(PendingReplicationBlocks.java:186)
> java.lang.Thread.run(Thread.java:619)
> Thread 9 (RMI TCP Accept-0):
> State: RUNNABLE
> Blocked count: 0
> Waited count: 0
> Stack:
> java.net.PlainSocketImpl.socketAccept(Native Method)
> java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
> java.net.ServerSocket.implAccept(ServerSocket.java:453)
> java.net.ServerSocket.accept(ServerSocket.java:421)
>
> sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTrans
> port.java:369)
>
> sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
> java.lang.Thread.run(Thread.java:619)
> Thread 4 (Signal Dispatcher):
> State: RUNNABLE
> Blocked count: 0
> Waited count: 0
> Stack:
> Thread 3 (Finalizer):
> State: WAITING
> Blocked count: 0
> Waited count: 40
> Waiting on java.lang.ref.ReferenceQueue$Lock@22f62eba
> Stack:
> java.lang.Object.wait(Native Method)
> java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
> java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
> java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
> Thread 2 (Reference Handler):
> State: WAITING
> Blocked count: 1
> Waited count: 39
> Waiting on java.lang.ref.Reference$Lock@646d6aa0
> Stack:
> java.lang.Object.wait(Native Method)
> java.lang.Object.wait(Object.java:485)
> java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
> Thread 1 (main):
> State: WAITING
> Blocked count: 10
> Waited count: 10
> Waiting on org.apache.hadoop.ipc.RPC$Server@41f6321
> Stack:
> java.lang.Object.wait(Native Method)
> java.lang.Object.wait(Object.java:485)
> org.apache.hadoop.ipc.Server.join(Server.java:1122)
>
> org.apache.hadoop.hdfs.server.namenode.NameNode.join(NameNode.java:292)
>
> org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:966)
>
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 4:15 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> add into hadoop-env.sh the opts to the jdk-call. The logs should be
> accessible at he hadoop-log-directory.
>
> Also check http://jobtracker:50030/stacks - thats the same as jstack
> (jstack
> PID). Also you can use jstack -F PID to get a corefile (similar to
> /stacks I
> think) @jobtracker.
>
> Are you using 64bit-JDK? Which version?
>
> regards,
> Alex
>
> On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com> wrote:
>
> > I am using the default heap size, which is 1000MB. The jobtracker hung
> > when only I was running one job. Now I could not even restart the
> > jobtracker.
> > Can you teach me how to turn on GC logging in hadoop?
> >
> > Thanks!
> > Wei
> >
> > -----Original Message-----
> > From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> > Sent: Friday, October 21, 2011 3:54 AM
> > To: common-user@hadoop.apache.org
> > Subject: Re: jobtracker cannot be started
> >
> > Hi,
> >
> > what are the heap size you given at the jobtracker? And how much jobs
> /
> > users / tasks are run? What say a log?
> > Turn on GC logging:
> > http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
> >
> > - Alex
> >
> >
> > On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com> wrote:
> >
> > > Hi,
> > >
> > >
> > >
> > > When I was running a job on hadoop with 75% mappers finished, the
> > > jobtracker hung so that I cannot access
> > > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
> > > well.
> > >
> > >
> > >
> > > Then I stopped jobtracker and restarted it. However, the jobtracker
> > > cannot be started. I received error message from jobtracker.log.out
> > > saying
> > >
> > >
> > >
> > > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java
> > heap
> > > space
> > >
> > > at
> > > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> > >
> > > at
> > > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> > >
> > > at
> > >
> >
> org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> > >
> > > at
> > > org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> > >
> > > at
> org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> > >
> > > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> > >
> > > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> > >
> > > at $Proxy4.renewLease(Unknown Source)
> > >
> > > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
> Source)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at
> > >
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > > cationHandler.java:82)
> > >
> > > at
> > >
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > > Handler.java:59)
> > >
> > > at $Proxy4.renewLease(Unknown Source)
> > >
> > > at
> > >
> >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> > >
> > > at
> > >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> > >
> > > at java.lang.Thread.run(Thread.java:619)
> > >
> > > Exception in thread "expireTrackers" java.lang.OutOfMemoryError:
> Java
> > > heap space
> > >
> > > at java.util.Arrays.copyOf(Arrays.java:2882)
> > >
> > > at
> > >
> >
> java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > > a:100)
> > >
> > > at
> > >
> java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> > >
> > > at java.lang.StringBuffer.append(StringBuffer.java:224)
> > >
> > > at
> org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> > >
> > > at
> > >
> >
> org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > > a:1354)
> > >
> > > at
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > > 2)
> > >
> > > at
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > > va:849)
> > >
> > > at
> > >
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > > 3)
> > >
> > > at
> > >
> >
> org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > > )
> > >
> > > at
> > >
> >
> org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > > 48)
> > >
> > > at java.lang.Thread.run(Thread.java:619)
> > >
> > > Exception in thread "IPC Server listener on 9001"
> > > java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > > va:587)
> > >
> > > at
> > >
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > > or.java:124)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > > 707)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:189)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> > > handler- the VM may need to be forcibly terminated
> > >
> > > Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java
> heap
> > > space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > > .java:281)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > > er.java:44)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > > va:213)
> > >
> > > at java.util.TimerThread.mainLoop(Timer.java:512)
> > >
> > > at java.util.TimerThread.run(Timer.java:462)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > Exception in thread "IPC Server Responder"
> java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 429)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > Exception in thread "expireLaunchingTasks"
> java.lang.OutOfMemoryError:
> > > Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:189)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 429)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > > 16)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> > >
> > > at
> > >
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > > Collection.java:230)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> > >
> > > at org.mortbay.jetty.Server.handle(Server.java:324)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > > java.lang.reflect.InvocationTargetException
> > >
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >
> > > at
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > > a:39)
> > >
> > > at
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > > Impl.java:25)
> > >
> > > at java.lang.reflect.Method.invoke(Method.java:597)
> > >
> > > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> > >
> > > at org.mortbay.log.Log.warn(Log.java:189)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> > >
> > > at
> > >
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > > ction.java:864)
> > >
> > > at
> org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> > >
> > > at
> > > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> > >
> > > at
> > > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> > >
> > > at
> > >
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > > 409)
> > >
> > > at
> > >
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > > :522)
> > >
> > > Caused by: java.lang.OutOfMemoryError: Java heap space
> > >
> > >
> > >
> > > I have no idea how to solve this problem. The memory on the
> jobtracker
> > > server seems ok
> > >
> > > total used free shared buffers
> > > cached
> > >
> > > Mem: 7148044 3578024 3570020 0 673768
> > > 467340
> > >
> > > -/+ buffers/cache: 2436916 4711128
> > >
> > > Swap: 1020116 160 1019956
> > >
> > >
> > >
> > > Can someone help me on this?
> > >
> > >
> > >
> > > Thanks a lot
> > >
> > > Wei
> > >
> > >
> >
> >
> > --
> > Alexander Lorenz
> > http://mapredit.blogspot.com
> >
>
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
RE: jobtracker cannot be started
Posted by "Peng, Wei" <We...@xerox.com>.
Thank you for your quick reply!!
I cannot change the hadoop conf files because they are owned by a person
who has left the company, though I have the root access. My Java version
is java version "1.5.0_07"
Java(TM) 2 Runtime Environment, Standard Edition (build 1.5.0_07-b03)
Java HotSpot(TM) 64-Bit Server VM (build 1.5.0_07-b03, mixed mode)
The log on http://jobtracker:50030/stacks is
Process Thread Dump:
26 active threads
Thread 53 (1424598978@qtp0-5):
State: RUNNABLE
Blocked count: 0
Waited count: 29
Stack:
sun.management.ThreadImpl.getThreadInfo0(Native Method)
sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.j
ava:149)
org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:505
)
javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:363)
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
16)
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
Collection.java:230)
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
org.mortbay.jetty.Server.handle(Server.java:324)
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
ction.java:864)
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
Thread 43 (Trash Emptier):
State: TIMED_WAITING
Blocked count: 0
Waited count: 183
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.fs.Trash$Emptier.run(Trash.java:234)
java.lang.Thread.run(Thread.java:619)
Thread 36 (IPC Server handler 9 on 9000):
State: WAITING
Blocked count: 32
Waited count: 3444
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 35 (IPC Server handler 8 on 9000):
State: WAITING
Blocked count: 29
Waited count: 3446
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 34 (IPC Server handler 7 on 9000):
State: WAITING
Blocked count: 30
Waited count: 3451
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 33 (IPC Server handler 6 on 9000):
State: WAITING
Blocked count: 33
Waited count: 3445
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 32 (IPC Server handler 5 on 9000):
State: WAITING
Blocked count: 29
Waited count: 3448
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 31 (IPC Server handler 4 on 9000):
State: WAITING
Blocked count: 33
Waited count: 3446
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 30 (IPC Server handler 3 on 9000):
State: WAITING
Blocked count: 29
Waited count: 3450
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 29 (IPC Server handler 2 on 9000):
State: WAITING
Blocked count: 29
Waited count: 3449
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 28 (IPC Server handler 1 on 9000):
State: WAITING
Blocked count: 36
Waited count: 3445
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 27 (IPC Server handler 0 on 9000):
State: WAITING
Blocked count: 44
Waited count: 3445
Waiting on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@49
59d87f
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.aw
ait(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:3
58)
org.apache.hadoop.ipc.Server$Handler.run(Server.java:939)
Thread 12 (IPC Server listener on 9000):
State: RUNNABLE
Blocked count: 5
Waited count: 0
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.ipc.Server$Listener.run(Server.java:318)
Thread 13 (IPC Server Responder):
State: RUNNABLE
Blocked count: 1
Waited count: 0
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.apache.hadoop.ipc.Server$Responder.run(Server.java:478)
Thread 26 (Timer-0):
State: TIMED_WAITING
Blocked count: 0
Waited count: 190
Stack:
java.lang.Object.wait(Native Method)
java.util.TimerThread.mainLoop(Timer.java:509)
java.util.TimerThread.run(Timer.java:462)
Thread 25 (1947740113@qtp0-1 - Acceptor0
SelectChannelConnector@penguin33:50070):
State: RUNNABLE
Blocked count: 579
Waited count: 5
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:184)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
va:429)
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
or.java:124)
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
707)
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
:522)
Thread 22
(org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@67c7
980c):
State: TIMED_WAITING
Blocked count: 5
Waited count: 127
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(D
ecommissionManager.java:65)
java.lang.Thread.run(Thread.java:619)
Thread 21
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@
2094257f):
State: TIMED_WAITING
Blocked count: 20
Waited count: 1263
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.r
un(FSNamesystem.java:2304)
java.lang.Thread.run(Thread.java:619)
Thread 20
(org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@3a51127a):
State: TIMED_WAITING
Blocked count: 21
Waited count: 1875
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseMan
ager.java:349)
java.lang.Thread.run(Thread.java:619)
Thread 19
(org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@61
578aab):
State: TIMED_WAITING
Blocked count: 0
Waited count: 13
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run
(FSNamesystem.java:2286)
java.lang.Thread.run(Thread.java:619)
Thread 18
(org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$Pending
ReplicationMonitor@2339e351):
State: TIMED_WAITING
Blocked count: 0
Waited count: 13
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingR
eplicationMonitor.run(PendingReplicationBlocks.java:186)
java.lang.Thread.run(Thread.java:619)
Thread 9 (RMI TCP Accept-0):
State: RUNNABLE
Blocked count: 0
Waited count: 0
Stack:
java.net.PlainSocketImpl.socketAccept(Native Method)
java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384)
java.net.ServerSocket.implAccept(ServerSocket.java:453)
java.net.ServerSocket.accept(ServerSocket.java:421)
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTrans
port.java:369)
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
java.lang.Thread.run(Thread.java:619)
Thread 4 (Signal Dispatcher):
State: RUNNABLE
Blocked count: 0
Waited count: 0
Stack:
Thread 3 (Finalizer):
State: WAITING
Blocked count: 0
Waited count: 40
Waiting on java.lang.ref.ReferenceQueue$Lock@22f62eba
Stack:
java.lang.Object.wait(Native Method)
java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
Thread 2 (Reference Handler):
State: WAITING
Blocked count: 1
Waited count: 39
Waiting on java.lang.ref.Reference$Lock@646d6aa0
Stack:
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:485)
java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
Thread 1 (main):
State: WAITING
Blocked count: 10
Waited count: 10
Waiting on org.apache.hadoop.ipc.RPC$Server@41f6321
Stack:
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:485)
org.apache.hadoop.ipc.Server.join(Server.java:1122)
org.apache.hadoop.hdfs.server.namenode.NameNode.join(NameNode.java:292)
org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:966)
Wei
-----Original Message-----
From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
Sent: Friday, October 21, 2011 4:15 AM
To: common-user@hadoop.apache.org
Subject: Re: jobtracker cannot be started
add into hadoop-env.sh the opts to the jdk-call. The logs should be
accessible at he hadoop-log-directory.
Also check http://jobtracker:50030/stacks - thats the same as jstack
(jstack
PID). Also you can use jstack -F PID to get a corefile (similar to
/stacks I
think) @jobtracker.
Are you using 64bit-JDK? Which version?
regards,
Alex
On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com> wrote:
> I am using the default heap size, which is 1000MB. The jobtracker hung
> when only I was running one job. Now I could not even restart the
> jobtracker.
> Can you teach me how to turn on GC logging in hadoop?
>
> Thanks!
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 3:54 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> Hi,
>
> what are the heap size you given at the jobtracker? And how much jobs
/
> users / tasks are run? What say a log?
> Turn on GC logging:
> http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
>
> - Alex
>
>
> On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com> wrote:
>
> > Hi,
> >
> >
> >
> > When I was running a job on hadoop with 75% mappers finished, the
> > jobtracker hung so that I cannot access
> > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
> > well.
> >
> >
> >
> > Then I stopped jobtracker and restarted it. However, the jobtracker
> > cannot be started. I received error message from jobtracker.log.out
> > saying
> >
> >
> >
> > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java
> heap
> > space
> >
> > at
> > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> >
> > at
> > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> >
> > at
> >
>
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> >
> > at
> > org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> >
> > at
org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> >
> > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> >
> > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> >
> > at $Proxy4.renewLease(Unknown Source)
> >
> > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
Source)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > cationHandler.java:82)
> >
> > at
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > Handler.java:59)
> >
> > at $Proxy4.renewLease(Unknown Source)
> >
> > at
> >
>
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> >
> > at
> >
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> >
> > at java.lang.Thread.run(Thread.java:619)
> >
> > Exception in thread "expireTrackers" java.lang.OutOfMemoryError:
Java
> > heap space
> >
> > at java.util.Arrays.copyOf(Arrays.java:2882)
> >
> > at
> >
>
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > a:100)
> >
> > at
> >
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> >
> > at java.lang.StringBuffer.append(StringBuffer.java:224)
> >
> > at
org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > a:1354)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > 2)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > va:849)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > 3)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > )
> >
> > at
> >
>
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > 48)
> >
> > at java.lang.Thread.run(Thread.java:619)
> >
> > Exception in thread "IPC Server listener on 9001"
> > java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> >
> > at
> >
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > va:587)
> >
> > at
> >
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> >
> > at
> >
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > or.java:124)
> >
> > at
> >
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > 707)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> > handler- the VM may need to be forcibly terminated
> >
> > Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java
heap
> > space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > .java:281)
> >
> > at
> >
>
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > er.java:44)
> >
> > at
> >
>
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > va:213)
> >
> > at java.util.TimerThread.mainLoop(Timer.java:512)
> >
> > at java.util.TimerThread.run(Timer.java:462)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Exception in thread "IPC Server Responder"
java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 429)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Exception in thread "expireLaunchingTasks"
java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 429)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> >
> >
> > I have no idea how to solve this problem. The memory on the
jobtracker
> > server seems ok
> >
> > total used free shared buffers
> > cached
> >
> > Mem: 7148044 3578024 3570020 0 673768
> > 467340
> >
> > -/+ buffers/cache: 2436916 4711128
> >
> > Swap: 1020116 160 1019956
> >
> >
> >
> > Can someone help me on this?
> >
> >
> >
> > Thanks a lot
> >
> > Wei
> >
> >
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
RE: jobtracker cannot be started
Posted by "Peng, Wei" <We...@xerox.com>.
The jobtracker was started successfully after rebooting the server !!
Thank you for all your help!!
Wei
-----Original Message-----
From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
Sent: Friday, October 21, 2011 1:15 AM
To: common-user@hadoop.apache.org
Subject: Re: jobtracker cannot be started
add into hadoop-env.sh the opts to the jdk-call. The logs should be
accessible at he hadoop-log-directory.
Also check http://jobtracker:50030/stacks - thats the same as jstack
(jstack
PID). Also you can use jstack -F PID to get a corefile (similar to
/stacks I
think) @jobtracker.
Are you using 64bit-JDK? Which version?
regards,
Alex
On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com> wrote:
> I am using the default heap size, which is 1000MB. The jobtracker hung
> when only I was running one job. Now I could not even restart the
> jobtracker.
> Can you teach me how to turn on GC logging in hadoop?
>
> Thanks!
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 3:54 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> Hi,
>
> what are the heap size you given at the jobtracker? And how much jobs
/
> users / tasks are run? What say a log?
> Turn on GC logging:
> http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
>
> - Alex
>
>
> On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com> wrote:
>
> > Hi,
> >
> >
> >
> > When I was running a job on hadoop with 75% mappers finished, the
> > jobtracker hung so that I cannot access
> > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
> > well.
> >
> >
> >
> > Then I stopped jobtracker and restarted it. However, the jobtracker
> > cannot be started. I received error message from jobtracker.log.out
> > saying
> >
> >
> >
> > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java
> heap
> > space
> >
> > at
> > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> >
> > at
> > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> >
> > at
> >
>
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> >
> > at
> > org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> >
> > at
org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> >
> > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> >
> > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> >
> > at $Proxy4.renewLease(Unknown Source)
> >
> > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown
Source)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > cationHandler.java:82)
> >
> > at
> >
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > Handler.java:59)
> >
> > at $Proxy4.renewLease(Unknown Source)
> >
> > at
> >
>
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> >
> > at
> >
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> >
> > at java.lang.Thread.run(Thread.java:619)
> >
> > Exception in thread "expireTrackers" java.lang.OutOfMemoryError:
Java
> > heap space
> >
> > at java.util.Arrays.copyOf(Arrays.java:2882)
> >
> > at
> >
>
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > a:100)
> >
> > at
> >
java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> >
> > at java.lang.StringBuffer.append(StringBuffer.java:224)
> >
> > at
org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > a:1354)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > 2)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > va:849)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > 3)
> >
> > at
> >
>
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > )
> >
> > at
> >
>
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > 48)
> >
> > at java.lang.Thread.run(Thread.java:619)
> >
> > Exception in thread "IPC Server listener on 9001"
> > java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> >
> > at
> >
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > va:587)
> >
> > at
> >
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> >
> > at
> >
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > or.java:124)
> >
> > at
> >
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > 707)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> > handler- the VM may need to be forcibly terminated
> >
> > Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java
heap
> > space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > .java:281)
> >
> > at
> >
>
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > er.java:44)
> >
> > at
> >
>
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > va:213)
> >
> > at java.util.TimerThread.mainLoop(Timer.java:512)
> >
> > at java.util.TimerThread.run(Timer.java:462)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Exception in thread "IPC Server Responder"
java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 429)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Exception in thread "expireLaunchingTasks"
java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 429)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> >
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
> >
> > at
> >
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at
org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> >
> >
> > I have no idea how to solve this problem. The memory on the
jobtracker
> > server seems ok
> >
> > total used free shared buffers
> > cached
> >
> > Mem: 7148044 3578024 3570020 0 673768
> > 467340
> >
> > -/+ buffers/cache: 2436916 4711128
> >
> > Swap: 1020116 160 1019956
> >
> >
> >
> > Can someone help me on this?
> >
> >
> >
> > Thanks a lot
> >
> > Wei
> >
> >
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
Re: jobtracker cannot be started
Posted by "Alexander C.H. Lorenz" <wg...@googlemail.com>.
add into hadoop-env.sh the opts to the jdk-call. The logs should be
accessible at he hadoop-log-directory.
Also check http://jobtracker:50030/stacks - thats the same as jstack (jstack
PID). Also you can use jstack -F PID to get a corefile (similar to /stacks I
think) @jobtracker.
Are you using 64bit-JDK? Which version?
regards,
Alex
On Fri, Oct 21, 2011 at 10:00 AM, Peng, Wei <We...@xerox.com> wrote:
> I am using the default heap size, which is 1000MB. The jobtracker hung
> when only I was running one job. Now I could not even restart the
> jobtracker.
> Can you teach me how to turn on GC logging in hadoop?
>
> Thanks!
> Wei
>
> -----Original Message-----
> From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
> Sent: Friday, October 21, 2011 3:54 AM
> To: common-user@hadoop.apache.org
> Subject: Re: jobtracker cannot be started
>
> Hi,
>
> what are the heap size you given at the jobtracker? And how much jobs /
> users / tasks are run? What say a log?
> Turn on GC logging:
> http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
>
> - Alex
>
>
> On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com> wrote:
>
> > Hi,
> >
> >
> >
> > When I was running a job on hadoop with 75% mappers finished, the
> > jobtracker hung so that I cannot access
> > jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
> > well.
> >
> >
> >
> > Then I stopped jobtracker and restarted it. However, the jobtracker
> > cannot be started. I received error message from jobtracker.log.out
> > saying
> >
> >
> >
> > Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java
> heap
> > space
> >
> > at
> > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
> >
> > at
> > java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
> >
> > at
> >
> org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
> >
> > at
> > org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
> >
> > at org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
> >
> > at org.apache.hadoop.ipc.Client.call(Client.java:719)
> >
> > at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
> >
> > at $Proxy4.renewLease(Unknown Source)
> >
> > at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> > cationHandler.java:82)
> >
> > at
> >
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> > Handler.java:59)
> >
> > at $Proxy4.renewLease(Unknown Source)
> >
> > at
> >
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
> >
> > at
> > org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
> >
> > at java.lang.Thread.run(Thread.java:619)
> >
> > Exception in thread "expireTrackers" java.lang.OutOfMemoryError: Java
> > heap space
> >
> > at java.util.Arrays.copyOf(Arrays.java:2882)
> >
> > at
> >
> java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> > a:100)
> >
> > at
> > java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
> >
> > at java.lang.StringBuffer.append(StringBuffer.java:224)
> >
> > at org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
> >
> > at
> >
> org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> > a:1354)
> >
> > at
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> > 2)
> >
> > at
> >
> org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> > va:849)
> >
> > at
> >
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> > 3)
> >
> > at
> >
> org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> > )
> >
> > at
> >
> org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> > 48)
> >
> > at java.lang.Thread.run(Thread.java:619)
> >
> > Exception in thread "IPC Server listener on 9001"
> > java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at org.mortbay.thread.Timeout.tick(Timeout.java:159)
> >
> > at
> >
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> > va:587)
> >
> > at
> > org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
> >
> > at
> >
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> > or.java:124)
> >
> > at
> >
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> > 707)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Java HotSpot(TM) 64-Bit Server VM warning: Exception
> > java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> > handler- the VM may need to be forcibly terminated
> >
> > Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java heap
> > space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
> org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> > .java:281)
> >
> > at
> >
> org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> > er.java:44)
> >
> > at
> >
> org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> > va:213)
> >
> > at java.util.TimerThread.mainLoop(Timer.java:512)
> >
> > at java.util.TimerThread.run(Timer.java:462)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Exception in thread "IPC Server Responder" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 429)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > Exception in thread "expireLaunchingTasks" java.lang.OutOfMemoryError:
> > Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 429)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:181)
> >
> > at
> >
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
> >
> > at
> >
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> > 16)
> >
> > at
> >
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
> >
> > at
> > org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
> >
> > at
> >
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> > Collection.java:230)
> >
> > at
> >
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> >
> > at org.mortbay.jetty.Server.handle(Server.java:324)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> > java.lang.reflect.InvocationTargetException
> >
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> > a:39)
> >
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> > Impl.java:25)
> >
> > at java.lang.reflect.Method.invoke(Method.java:597)
> >
> > at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
> >
> > at org.mortbay.log.Log.warn(Log.java:189)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
> >
> > at
> >
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> > ction.java:864)
> >
> > at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
> >
> > at
> > org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
> >
> > at
> > org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
> >
> > at
> >
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> > 409)
> >
> > at
> >
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> > :522)
> >
> > Caused by: java.lang.OutOfMemoryError: Java heap space
> >
> >
> >
> > I have no idea how to solve this problem. The memory on the jobtracker
> > server seems ok
> >
> > total used free shared buffers
> > cached
> >
> > Mem: 7148044 3578024 3570020 0 673768
> > 467340
> >
> > -/+ buffers/cache: 2436916 4711128
> >
> > Swap: 1020116 160 1019956
> >
> >
> >
> > Can someone help me on this?
> >
> >
> >
> > Thanks a lot
> >
> > Wei
> >
> >
>
>
> --
> Alexander Lorenz
> http://mapredit.blogspot.com
>
--
Alexander Lorenz
http://mapredit.blogspot.com
RE: jobtracker cannot be started
Posted by "Peng, Wei" <We...@xerox.com>.
I am using the default heap size, which is 1000MB. The jobtracker hung
when only I was running one job. Now I could not even restart the
jobtracker.
Can you teach me how to turn on GC logging in hadoop?
Thanks!
Wei
-----Original Message-----
From: Alexander C.H. Lorenz [mailto:wget.null@googlemail.com]
Sent: Friday, October 21, 2011 3:54 AM
To: common-user@hadoop.apache.org
Subject: Re: jobtracker cannot be started
Hi,
what are the heap size you given at the jobtracker? And how much jobs /
users / tasks are run? What say a log?
Turn on GC logging:
http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
- Alex
On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com> wrote:
> Hi,
>
>
>
> When I was running a job on hadoop with 75% mappers finished, the
> jobtracker hung so that I cannot access
> jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
> well.
>
>
>
> Then I stopped jobtracker and restarted it. However, the jobtracker
> cannot be started. I received error message from jobtracker.log.out
> saying
>
>
>
> Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java
heap
> space
>
> at
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
>
> at
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
>
> at
>
org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
>
> at
> org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
>
> at org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
>
> at org.apache.hadoop.ipc.Client.call(Client.java:719)
>
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
>
> at $Proxy4.renewLease(Unknown Source)
>
> at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> cationHandler.java:82)
>
> at
>
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> Handler.java:59)
>
> at $Proxy4.renewLease(Unknown Source)
>
> at
>
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
>
> at
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
>
> at java.lang.Thread.run(Thread.java:619)
>
> Exception in thread "expireTrackers" java.lang.OutOfMemoryError: Java
> heap space
>
> at java.util.Arrays.copyOf(Arrays.java:2882)
>
> at
>
java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> a:100)
>
> at
> java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
>
> at java.lang.StringBuffer.append(StringBuffer.java:224)
>
> at org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
>
> at
>
org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> a:1354)
>
> at
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> 2)
>
> at
>
org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> va:849)
>
> at
>
org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> 3)
>
> at
>
org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> )
>
> at
>
org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> 48)
>
> at java.lang.Thread.run(Thread.java:619)
>
> Exception in thread "IPC Server listener on 9001"
> java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at org.mortbay.thread.Timeout.tick(Timeout.java:159)
>
> at
>
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> va:587)
>
> at
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
>
> at
>
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> or.java:124)
>
> at
>
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> 707)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:189)
>
> at
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
>
> at
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> Java HotSpot(TM) 64-Bit Server VM warning: Exception
> java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> handler- the VM may need to be forcibly terminated
>
> Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java heap
> space
>
> java.lang.reflect.InvocationTargetException
>
> Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
>
org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> .java:281)
>
> at
>
org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> er.java:44)
>
> at
>
org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> va:213)
>
> at java.util.TimerThread.mainLoop(Timer.java:512)
>
> at java.util.TimerThread.run(Timer.java:462)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> Exception in thread "IPC Server Responder" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 429)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> Exception in thread "expireLaunchingTasks" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:189)
>
> at
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
>
> at
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 429)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
>
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
>
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
>
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
>
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
>
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
>
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
>
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
>
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:189)
>
> at
>
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
>
> at
>
org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
>
org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
>
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
>
>
> I have no idea how to solve this problem. The memory on the jobtracker
> server seems ok
>
> total used free shared buffers
> cached
>
> Mem: 7148044 3578024 3570020 0 673768
> 467340
>
> -/+ buffers/cache: 2436916 4711128
>
> Swap: 1020116 160 1019956
>
>
>
> Can someone help me on this?
>
>
>
> Thanks a lot
>
> Wei
>
>
--
Alexander Lorenz
http://mapredit.blogspot.com
Re: jobtracker cannot be started
Posted by "Alexander C.H. Lorenz" <wg...@googlemail.com>.
Hi,
what are the heap size you given at the jobtracker? And how much jobs /
users / tasks are run? What say a log?
Turn on GC logging:
http://java.sun.com/developer/technicalArticles/Programming/GCPortal/
- Alex
On Fri, Oct 21, 2011 at 9:47 AM, Peng, Wei <We...@xerox.com> wrote:
> Hi,
>
>
>
> When I was running a job on hadoop with 75% mappers finished, the
> jobtracker hung so that I cannot access
> jobtrackerserver:7845/jobtracker.jsp and hadoop job -status hung as
> well.
>
>
>
> Then I stopped jobtracker and restarted it. However, the jobtracker
> cannot be started. I received error message from jobtracker.log.out
> saying
>
>
>
> Exception in thread "LeaseChecker" java.lang.OutOfMemoryError: Java heap
> space
>
> at
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:59)
>
> at
> java.io.BufferedOutputStream.<init>(BufferedOutputStream.java:42)
>
> at
> org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:318)
>
> at
> org.apache.hadoop.ipc.Client$Connection.access$1700(Client.java:176)
>
> at org.apache.hadoop.ipc.Client.getConnection(Client.java:859)
>
> at org.apache.hadoop.ipc.Client.call(Client.java:719)
>
> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:220)
>
> at $Proxy4.renewLease(Unknown Source)
>
> at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvo
> cationHandler.java:82)
>
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocation
> Handler.java:59)
>
> at $Proxy4.renewLease(Unknown Source)
>
> at
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.renew(DFSClient.java:1016)
>
> at
> org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1028)
>
> at java.lang.Thread.run(Thread.java:619)
>
> Exception in thread "expireTrackers" java.lang.OutOfMemoryError: Java
> heap space
>
> at java.util.Arrays.copyOf(Arrays.java:2882)
>
> at
> java.lang.AbstractStringBuilder.expandCapacity(AbstractStringBuilder.jav
> a:100)
>
> at
> java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:390)
>
> at java.lang.StringBuffer.append(StringBuffer.java:224)
>
> at org.apache.hadoop.mapred.JobHistory.log(JobHistory.java:354)
>
> at
> org.apache.hadoop.mapred.JobHistory$MapAttempt.logStarted(JobHistory.jav
> a:1354)
>
> at
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:233
> 2)
>
> at
> org.apache.hadoop.mapred.JobInProgress.updateTaskStatus(JobInProgress.ja
> va:849)
>
> at
> org.apache.hadoop.mapred.JobInProgress.failedTask(JobInProgress.java:246
> 3)
>
> at
> org.apache.hadoop.mapred.JobTracker.lostTaskTracker(JobTracker.java:3474
> )
>
> at
> org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:3
> 48)
>
> at java.lang.Thread.run(Thread.java:619)
>
> Exception in thread "IPC Server listener on 9001"
> java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at org.mortbay.thread.Timeout.tick(Timeout.java:159)
>
> at
> org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.ja
> va:587)
>
> at
> org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185)
>
> at
> org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnect
> or.java:124)
>
> at
> org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:
> 707)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:189)
>
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> Java HotSpot(TM) 64-Bit Server VM warning: Exception
> java.lang.OutOfMemoryError occurred dispatching signal SIGTERM to
> handler- the VM may need to be forcibly terminated
>
> Exception in thread "retireJobs" java.lang.OutOfMemoryError: Java heap
> space
>
> java.lang.reflect.InvocationTargetException
>
> Exception in thread "1586500830@qtp0-37" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> Exception in thread "1513282332@qtp0-29" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
> org.mortbay.jetty.servlet.HashSessionManager.scavenge(HashSessionManager
> .java:281)
>
> at
> org.mortbay.jetty.servlet.HashSessionManager.access$000(HashSessionManag
> er.java:44)
>
> at
> org.mortbay.jetty.servlet.HashSessionManager$2.run(HashSessionManager.ja
> va:213)
>
> at java.util.TimerThread.mainLoop(Timer.java:512)
>
> at java.util.TimerThread.run(Timer.java:462)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> Exception in thread "IPC Server Responder" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 429)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> Exception in thread "expireLaunchingTasks" java.lang.OutOfMemoryError:
> Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:189)
>
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 429)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:181)
>
> at
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:449)
>
> at
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:2
> 16)
>
> at
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
>
> at
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
>
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
>
> at
> org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandler
> Collection.java:230)
>
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
>
> at org.mortbay.jetty.Server.handle(Server.java:324)
>
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
> java.lang.reflect.InvocationTargetException
>
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.jav
> a:39)
>
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessor
> Impl.java:25)
>
> at java.lang.reflect.Method.invoke(Method.java:597)
>
> at org.mortbay.log.Slf4jLog.warn(Slf4jLog.java:126)
>
> at org.mortbay.log.Log.warn(Log.java:189)
>
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:563)
>
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConne
> ction.java:864)
>
> at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:533)
>
> at
> org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:207)
>
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:403)
>
> at
> org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:
> 409)
>
> at
> org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java
> :522)
>
> Caused by: java.lang.OutOfMemoryError: Java heap space
>
>
>
> I have no idea how to solve this problem. The memory on the jobtracker
> server seems ok
>
> total used free shared buffers
> cached
>
> Mem: 7148044 3578024 3570020 0 673768
> 467340
>
> -/+ buffers/cache: 2436916 4711128
>
> Swap: 1020116 160 1019956
>
>
>
> Can someone help me on this?
>
>
>
> Thanks a lot
>
> Wei
>
>
--
Alexander Lorenz
http://mapredit.blogspot.com