You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@flume.apache.org by Sutanu Das <sd...@att.com> on 2015/10/28 19:59:52 UTC

Sinks stop processing with GC errors

Team, My hdfs sinks stop processing and I see there GC errors in Flume log

Any ideas? Please help.


ary.path= org.apache.flume.node.Application --conf-file /etc/flume/conf/airwave_xml.conf --name airwave_xml start
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded
        at java.lang.ClassLoader.findLoadedClass0(Native Method)
        at java.lang.ClassLoader.findLoadedClass(ClassLoader.java:1093)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:407)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
        at java.lang.ClassLoader.defineClass1(Native Method)
        at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
        at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
        at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
        at org.apache.log4j.spi.ThrowableInformation.getThrowableStrRep(ThrowableInformation.java:87)
        at org.apache.log4j.spi.LoggingEvent.getThrowableStrRep(LoggingEvent.java:413)
        at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:313)
        at org.apache.log4j.RollingFileAppender.subAppend(RollingFileAppender.java:276)
        at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
        at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
        at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
        at org.apache.log4j.Category.callAppenders(Category.java:206)
        at org.apache.log4j.Category.forcedLog(Category.java:391)
        at org.apache.log4j.Category.log(Category.java:856)
        at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
        at org.apache.flume.sink.hdfs.HDFSEventSink.process(HDFSEventSink.java:466)
        at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)
        at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)
        at java.lang.Thread.run(Thread.java:744)
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded

Re: Sinks stop processing with GC errors

Posted by "Bhaskar V. Karambelkar" <bh...@gmail.com>.
This is a classic example of how not to ask a question.
You haven't told us anything that will help us debug this issue. For
starters what's the amount of RAM avail on your box, how much have you
allocated to the JVM, Did you try verbose GC and/or GC logging ?



On Wed, Oct 28, 2015 at 2:59 PM, Sutanu Das <sd...@att.com> wrote:

> Team, My hdfs sinks stop processing and I see there GC errors in Flume log
>
>
>
> Any ideas? Please help.
>
>
>
>
>
> ary.path= org.apache.flume.node.Application --conf-file
> /etc/flume/conf/airwave_xml.conf --name airwave_xml start
>
> Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor"
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>
>         at java.lang.ClassLoader.findLoadedClass0(Native Method)
>
>         at java.lang.ClassLoader.findLoadedClass(ClassLoader.java:1093)
>
>         at java.lang.ClassLoader.loadClass(ClassLoader.java:407)
>
>         at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
>
>         at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
>
>         at java.lang.ClassLoader.defineClass1(Native Method)
>
>         at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
>
>         at
> java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
>
>         at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
>
>         at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
>
>         at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
>
>         at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
>
>         at java.security.AccessController.doPrivileged(Native Method)
>
>         at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
>
>         at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
>
>         at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
>
>         at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
>
>         at
> org.apache.log4j.spi.ThrowableInformation.getThrowableStrRep(ThrowableInformation.java:87)
>
>         at
> org.apache.log4j.spi.LoggingEvent.getThrowableStrRep(LoggingEvent.java:413)
>
>         at
> org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:313)
>
>         at
> org.apache.log4j.RollingFileAppender.subAppend(RollingFileAppender.java:276)
>
>         at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
>
>         at
> org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
>
>         at
> org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
>
>         at org.apache.log4j.Category.callAppenders(Category.java:206)
>
>         at org.apache.log4j.Category.forcedLog(Category.java:391)
>
>         at org.apache.log4j.Category.log(Category.java:856)
>
>         at
> org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
>
>         at
> org.apache.flume.sink.hdfs.HDFSEventSink.process(HDFSEventSink.java:466)
>
>         at
> org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)
>
>         at
> org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)
>
>         at java.lang.Thread.run(Thread.java:744)
>
> Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor"
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>
> Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor"
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>
> Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor"
> java.lang.OutOfMemoryError: GC overhead limit exceeded
>

RE: Sinks stop processing with GC errors

Posted by Sutanu Das <sd...@att.com>.
Sorry, mybad, I changed the Xms/Xmx in flume-env.sh which I think will address the issue ?

From: Sutanu Das [mailto:sd2302@att.com]
Sent: Wednesday, October 28, 2015 2:00 PM
To: user@flume.apache.org
Subject: Sinks stop processing with GC errors

***Security Advisory: This Message Originated Outside of AT&T ***
Reference http://cso.att.com/EmailSecurity/IDSP.html for more information.
Team, My hdfs sinks stop processing and I see there GC errors in Flume log

Any ideas? Please help.


ary.path= org.apache.flume.node.Application --conf-file /etc/flume/conf/airwave_xml.conf --name airwave_xml start
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded
        at java.lang.ClassLoader.findLoadedClass0(Native Method)
        at java.lang.ClassLoader.findLoadedClass(ClassLoader.java:1093)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:407)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
        at java.lang.ClassLoader.defineClass1(Native Method)
        at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
        at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
        at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
        at org.apache.log4j.spi.ThrowableInformation.getThrowableStrRep(ThrowableInformation.java:87)
        at org.apache.log4j.spi.LoggingEvent.getThrowableStrRep(LoggingEvent.java:413)
        at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:313)
        at org.apache.log4j.RollingFileAppender.subAppend(RollingFileAppender.java:276)
        at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
        at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
        at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
        at org.apache.log4j.Category.callAppenders(Category.java:206)
        at org.apache.log4j.Category.forcedLog(Category.java:391)
        at org.apache.log4j.Category.log(Category.java:856)
        at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
        at org.apache.flume.sink.hdfs.HDFSEventSink.process(HDFSEventSink.java:466)
        at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:68)
        at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:147)
        at java.lang.Thread.run(Thread.java:744)
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "SinkRunner-PollingRunner-DefaultSinkProcessor" java.lang.OutOfMemoryError: GC overhead limit exceeded