You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hive.apache.org by 王锋 <wf...@163.com> on 2011/12/14 03:21:53 UTC

a serious issue of hiveserver

when hiveserver is executing sql, it throws an exception:


2011-12-14 00:40:43,762 Stage-1 map = 90%,  reduce = 13%
2011-12-14 00:40:45,771 Stage-1 map = 92%,  reduce = 13%
2011-12-14 00:40:46,777 Stage-1 map = 96%,  reduce = 13%
2011-12-14 00:40:47,782 Stage-1 map = 97%,  reduce = 13%
2011-12-14 00:40:48,787 Stage-1 map = 98%,  reduce = 13%
2011-12-14 00:40:49,791 Stage-1 map = 99%,  reduce = 13%
2011-12-14 00:40:50,796 Stage-1 map = 100%,  reduce = 13%
2011-12-14 00:40:51,800 Stage-1 map = 100%,  reduce = 30%
2011-12-14 00:40:58,830 Stage-1 map = 100%,  reduce = 100%
Ended Job = job_201111141853_635398
OK
Exception in thread "pool-1-thread-100" java.lang.StackOverflowError
    at java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
    at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
 at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
    at com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.findProviderClass(ObjectFactory.java:398)
    at com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.newInstance(ObjectFactory.java:353)
    at com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:62)
    at com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:47)
    at com.sun.org.apache.xerces.internal.parsers.XML11Configuration.<init>(XML11Configuration.java:565)
    at com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:130)
    at com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:91)
    at sun.reflect.GeneratedConstructorAccessor2.newInstance(Unknown Source)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
    at java.lang.Class.newInstance0(Class.java:372)
    at java.lang.Class.newInstance(Class.java:325)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.newInstance(ObjectFactory.java:352)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:157)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:100)
    at com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:138)
    at com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:122)
    at com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderImpl.<init>(DocumentBuilderImpl.java:120)
    at com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl.newDocumentBuilder(DocumentBuilderFactoryImpl.java:76)
    at org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:1282)
    at org.apache.hadoop.conf.Configuration.loadResources(Configuration.java:1251)
    at org.apache.hadoop.conf.Configuration.getProps(Configuration.java:1192)
    at org.apache.hadoop.conf.Configuration.set(Configuration.java:461)
    at org.apache.hadoop.mapred.JobConf.setJar(JobConf.java:426)
    at org.apache.hadoop.mapred.JobConf.setJarByClass(JobConf.java:444)
    at org.apache.hadoop.mapred.JobConf.<init>(JobConf.java:339)
    at org.apache.hadoop.hive.conf.HiveConf.initialize(HiveConf.java:617)
    at org.apache.hadoop.hive.conf.HiveConf.<init>(HiveConf.java:588)
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:190)
    at org.apache.hadoop.hive.service.HiveServer$HiveServerHandler.<init>(HiveServer.java:94)
    at org.apache.hadoop.hive.service.HiveServer$ThriftHiveProcessorFactory.getProcessor(HiveServer.java:503)
    at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:246)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
    at java.lang.Thread.run(Thread.java:722)
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140042_2056220006.txt
37860.048: [GC 37860.048: [ParNew: 2529200K->3304K(3686400K), 0.0133750 secs] 2634845K->108963K(4710400K), 0.0134870 secs] [Times: user=0.06 sys=0.00, real=0.02 secs]
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140043_1286096583.txt
37920.093: [GC 37920.093: [ParNew: 1918585K->3218K(3686400K), 0.0117880 secs] 2024244K->108877K(4710400K), 0.0118950 secs] [Times: user=0.04 sys=0.00, real=0.01 secs]
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140044_1787703059.txt
37980.126: [GC 37980.126: [ParNew: 1913955K->3238K(3686400K), 0.0128510 secs] 2019613K->108896K(4710400K), 0.0129470 secs] [Times: user=0.07 sys=0.00, real=0.02 secs]
Exception in thread "pool-1-thread-101" java.lang.StackOverflowError
    at java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
    at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)

Re:Re: a serious issue of hiveserver

Posted by 王锋 <wf...@163.com>.

The Exception “java.lang.StackOverflowError” should be depended on the Xss value . 


在 2011-12-22 16:15:41，"alo alt" <wg...@googlemail.com> 写道：
>Hi,
>
>depends on the memory you configured for hive (xms, xmx).
>
>- Alex
>
>2011/12/14 王锋 <wf...@163.com>:
>> when hiveserver is executing sql, it throws an exception:
>>
>> 2011-12-14 00:40:43,762 Stage-1 map = 90%,  reduce = 13%
>> 2011-12-14 00:40:45,771 Stage-1 map = 92%,  reduce = 13%
>> 2011-12-14 00:40:46,777 Stage-1 map = 96%,  reduce = 13%
>> 2011-12-14 00:40:47,782 Stage-1 map = 97%,  reduce = 13%
>> 2011-12-14 00:40:48,787 Stage-1 map = 98%,  reduce = 13%
>> 2011-12-14 00:40:49,791 Stage-1 map = 99%,  reduce = 13%
>> 2011-12-14 00:40:50,796 Stage-1 map = 100%,  reduce = 13%
>> 2011-12-14 00:40:51,800 Stage-1 map = 100%,  reduce = 30%
>> 2011-12-14 00:40:58,830 Stage-1 map = 100%,  reduce = 100%
>> Ended Job = job_201111141853_635398
>> OK
>> Exception in thread "pool-1-thread-100" java.lang.StackOverflowError
>>     at java.util.concurrent.Conc
>> urrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
>>     at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
>>     at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     a t java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.Class Loader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>  at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
>>     at
>> com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.findProviderClass(ObjectFactory.java:398)
>>     at
>> com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.newInstance(ObjectFactory.java:353)
>>     at
>> com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:62)
>>     at
>> com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:47)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.XML11Configuration.<init>(XML11Configuration.java:565)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:130)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:91)
>>     at sun.reflect.GeneratedConstructorAccessor2.newInstance(Unknown Source)
>>     at
>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>     at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
>>     at java.lang.Class.newInstance0(Class.java:372)
>>     at java.lang.Class.newInstanc e(Class.java:325)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.ObjectFactory.newInstance(ObjectFactory.java:352)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:157)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:100)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:138)
>>     at
>> com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:122)
>>     at
>> com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderImpl.<init>(DocumentBuilderImpl.java:120)
>>     at
>> com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl.newDocumentBuilder(DocumentBuilderFactoryImpl.java:76)
>>     at
>> org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:1282)
>>     at
>> org.apache.hadoop.conf.Configuration.loadResources(Configuration.java:1251)
>>     at
>> org.apache.hadoop.conf.Configuration.getProps(Configuration.java:1192)
>>     at org.apache.hadoop.conf.Configuration.set(Configuration.java:461)
>>     at org.apache.hadoop.mapred.JobConf.setJar(JobConf.java:426)
>>     at org.apache.hadoop.mapred.JobConf.setJarByClass(JobConf.java:444)
>>     at org.apache.hadoop.mapred.JobConf.<init>(JobConf.java:339)
>>     at org.apache.hadoop.hive.conf.HiveConf.initialize(HiveConf.java:617)
>>     at org.apache.hadoop.hive.conf.HiveConf.<init>(HiveConf.java:588)
>>     at
>> org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:190)
>>     at
>> org.apache.hadoop.hive.service.HiveServer$HiveServerHandler.<init>(HiveServer.java:94
>> )
>>     at
>> org.apache.hadoop.hive.service.HiveServer$ThriftHiveProcessorFactory.getProcessor(HiveServer.java:503)
>>     at
>> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:246)
>>     at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>>     at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>>     at java.lang.Thread.run(Thread.java:722)
>> Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140042_2056220006.txt
>> 37860.048: [GC 37860.048: [ParNew: 2529200K->3304K(3686400K), 0.0133750
>> secs] 2634845K->108963K(4710400K), 0.0134870 secs] [Times: user=0.06
>> sys=0.00, real=0.02 secs]
>> Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140043_1286096583.txt
>> 37920.093: [GC 37920.093: [ParNew: 1918585K->3218K(3686400K), 0.0117880
>> secs] 2024244K->108877K (4710400K), 0.0118950 secs] [Times: user=0.04
>> sys=0.00, real=0.01 secs]
>> Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140044_1787703059.txt
>> 37980.126: [GC 37980.126: [ParNew: 1913955K->3238K(3686400K), 0.0128510
>> secs] 2019613K->108896K(4710400K), 0.0129470 secs] [Times: user=0.07
>> sys=0.00, real=0.02 secs]
>> Exception in thread "pool-1-thread-101" java.lang.StackOverflowError
>>     at
>> java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
>>     at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader. loadClass(ClassLoader.java:410)
>>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>>
>>
>>
>>
>>
>
>
>
>-- 
>Alexander Lorenz
>http://mapredit.blogspot.com
>
>P Think of the environment: please don't print this email unless you
>really need to.

Re: a serious issue of hiveserver

Posted by alo alt <wg...@googlemail.com>.
Hi,

depends on the memory you configured for hive (xms, xmx).

- Alex

2011/12/14 王锋 <wf...@163.com>:
> when hiveserver is executing sql, it throws an exception:
>
> 2011-12-14 00:40:43,762 Stage-1 map = 90%,  reduce = 13%
> 2011-12-14 00:40:45,771 Stage-1 map = 92%,  reduce = 13%
> 2011-12-14 00:40:46,777 Stage-1 map = 96%,  reduce = 13%
> 2011-12-14 00:40:47,782 Stage-1 map = 97%,  reduce = 13%
> 2011-12-14 00:40:48,787 Stage-1 map = 98%,  reduce = 13%
> 2011-12-14 00:40:49,791 Stage-1 map = 99%,  reduce = 13%
> 2011-12-14 00:40:50,796 Stage-1 map = 100%,  reduce = 13%
> 2011-12-14 00:40:51,800 Stage-1 map = 100%,  reduce = 30%
> 2011-12-14 00:40:58,830 Stage-1 map = 100%,  reduce = 100%
> Ended Job = job_201111141853_635398
> OK
> Exception in thread "pool-1-thread-100" java.lang.StackOverflowError
>     at java.util.concurrent.Conc
> urrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
>     at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
>     at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     a t java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.Class Loader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>  at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
>     at
> com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.findProviderClass(ObjectFactory.java:398)
>     at
> com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.newInstance(ObjectFactory.java:353)
>     at
> com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:62)
>     at
> com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:47)
>     at
> com.sun.org.apache.xerces.internal.parsers.XML11Configuration.<init>(XML11Configuration.java:565)
>     at
> com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:130)
>     at
> com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:91)
>     at sun.reflect.GeneratedConstructorAccessor2.newInstance(Unknown Source)
>     at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>     at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
>     at java.lang.Class.newInstance0(Class.java:372)
>     at java.lang.Class.newInstanc e(Class.java:325)
>     at
> com.sun.org.apache.xerces.internal.parsers.ObjectFactory.newInstance(ObjectFactory.java:352)
>     at
> com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:157)
>     at
> com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:100)
>     at
> com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:138)
>     at
> com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:122)
>     at
> com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderImpl.<init>(DocumentBuilderImpl.java:120)
>     at
> com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl.newDocumentBuilder(DocumentBuilderFactoryImpl.java:76)
>     at
> org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:1282)
>     at
> org.apache.hadoop.conf.Configuration.loadResources(Configuration.java:1251)
>     at
> org.apache.hadoop.conf.Configuration.getProps(Configuration.java:1192)
>     at org.apache.hadoop.conf.Configuration.set(Configuration.java:461)
>     at org.apache.hadoop.mapred.JobConf.setJar(JobConf.java:426)
>     at org.apache.hadoop.mapred.JobConf.setJarByClass(JobConf.java:444)
>     at org.apache.hadoop.mapred.JobConf.<init>(JobConf.java:339)
>     at org.apache.hadoop.hive.conf.HiveConf.initialize(HiveConf.java:617)
>     at org.apache.hadoop.hive.conf.HiveConf.<init>(HiveConf.java:588)
>     at
> org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:190)
>     at
> org.apache.hadoop.hive.service.HiveServer$HiveServerHandler.<init>(HiveServer.java:94
> )
>     at
> org.apache.hadoop.hive.service.HiveServer$ThriftHiveProcessorFactory.getProcessor(HiveServer.java:503)
>     at
> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:246)
>     at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>     at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>     at java.lang.Thread.run(Thread.java:722)
> Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140042_2056220006.txt
> 37860.048: [GC 37860.048: [ParNew: 2529200K->3304K(3686400K), 0.0133750
> secs] 2634845K->108963K(4710400K), 0.0134870 secs] [Times: user=0.06
> sys=0.00, real=0.02 secs]
> Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140043_1286096583.txt
> 37920.093: [GC 37920.093: [ParNew: 1918585K->3218K(3686400K), 0.0117880
> secs] 2024244K->108877K (4710400K), 0.0118950 secs] [Times: user=0.04
> sys=0.00, real=0.01 secs]
> Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140044_1787703059.txt
> 37980.126: [GC 37980.126: [ParNew: 1913955K->3238K(3686400K), 0.0128510
> secs] 2019613K->108896K(4710400K), 0.0129470 secs] [Times: user=0.07
> sys=0.00, real=0.02 secs]
> Exception in thread "pool-1-thread-101" java.lang.StackOverflowError
>     at
> java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
>     at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader. loadClass(ClassLoader.java:410)
>     at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
>
>
>
>
>



-- 
Alexander Lorenz
http://mapredit.blogspot.com

P Think of the environment: please don't print this email unless you
really need to.

Re:Re: a serious issue of hiveserver

Posted by 王锋 <wf...@163.com>.
sorry， I forgot it.

It's because of the jvm method Stack .It was 128k before. we were sending hiveserver concurrent and max jobnum would be 10. Most of These jobs used udf to work, and udf used more mem because of recurrencion. Now we set Xss 512k, and hiveserver has been nomal.



在 2011-12-22 14:03:28，"Ping Zhu" <pi...@sharethis.com> 写道：
please discuss in English.


2011/12/21 王锋 <wf...@163.com>
这个原因知道了 是因为hiveserver jvm 方法栈设置低，之前设置128k，但是hiveserver里udf 类的函数需要递归 等，需求较大，修正为512k，已经稳定运行3天。

At 2011-12-14 10:21:53,"王锋" <wf...@163.com> wrote:

when hiveserver is executing sql, it throws an exception:


2011-12-14 00:40:43,762 Stage-1 map = 90%,  reduce = 13%
2011-12-14 00:40:45,771 Stage-1 map = 92%,  reduce = 13%
2011-12-14 00:40:46,777 Stage-1 map = 96%,  reduce = 13%
2011-12-14 00:40:47,782 Stage-1 map = 97%,  reduce = 13%
2011-12-14 00:40:48,787 Stage-1 map = 98%,  reduce = 13%
2011-12-14 00:40:49,791 Stage-1 map = 99%,  reduce = 13%
2011-12-14 00:40:50,796 Stage-1 map = 100%,  reduce = 13%
2011-12-14 00:40:51,800 Stage-1 map = 100%,  reduce = 30%
2011-12-14 00:40:58,830 Stage-1 map = 100%,  reduce = 100%
Ended Job = job_201111141853_635398
OK
Exception in thread "pool-1-thread-100" java.lang.StackOverflowError
    at java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
    at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
 at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
    at com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.findProviderClass(ObjectFactory.java:398)
    at com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.newInstance(ObjectFactory.java:353)
    at com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:62)
    at com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:47)
    at com.sun.org.apache.xerces.internal.parsers.XML11Configuration.<init>(XML11Configuration.java:565)
    at com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:130)
    at com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:91)
    at sun.reflect.GeneratedConstructorAccessor2.newInstance(Unknown Source)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
    at java.lang.Class.newInstance0(Class.java:372)
    at java.lang.Class.newInstance(Class.java:325)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.newInstance(ObjectFactory.java:352)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:157)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:100)
    at com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:138)
    at com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:122)
    at com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderImpl.<init>(DocumentBuilderImpl.java:120)
    at com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl.newDocumentBuilder(DocumentBuilderFactoryImpl.java:76)
    at org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:1282)
    at org.apache.hadoop.conf.Configuration.loadResources(Configuration.java:1251)
    at org.apache.hadoop.conf.Configuration.getProps(Configuration.java:1192)
    at org.apache.hadoop.conf.Configuration.set(Configuration.java:461)
    at org.apache.hadoop.mapred.JobConf.setJar(JobConf.java:426)
    at org.apache.hadoop.mapred.JobConf.setJarByClass(JobConf.java:444)
    at org.apache.hadoop.mapred.JobConf.<init>(JobConf.java:339)
    at org.apache.hadoop.hive.conf.HiveConf.initialize(HiveConf.java:617)
    at org.apache.hadoop.hive.conf.HiveConf.<init>(HiveConf.java:588)
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:190)
    at org.apache.hadoop.hive.service.HiveServer$HiveServerHandler.<init>(HiveServer.java:94)
    at org.apache.hadoop.hive.service.HiveServer$ThriftHiveProcessorFactory.getProcessor(HiveServer.java:503)
    at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:246)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
    at java.lang.Thread.run(Thread.java:722)
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140042_2056220006.txt
37860.048: [GC 37860.048: [ParNew: 2529200K->3304K(3686400K), 0.0133750 secs] 2634845K->108963K(4710400K), 0.0134870 secs] [Times: user=0.06 sys=0.00, real=0.02 secs]
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140043_1286096583.txt
37920.093: [GC 37920.093: [ParNew: 1918585K->3218K(3686400K), 0.0117880 secs] 2024244K->108877K(4710400K), 0.0118950 secs] [Times: user=0.04 sys=0.00, real=0.01 secs]
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140044_1787703059.txt
37980.126: [GC 37980.126: [ParNew: 1913955K->3238K(3686400K), 0.0128510 secs] 2019613K->108896K(4710400K), 0.0129470 secs] [Times: user=0.07 sys=0.00, real=0.02 secs]
Exception in thread "pool-1-thread-101" java.lang.StackOverflowError
    at java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
    at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)

Re:a serious issue of hiveserver

Posted by 王锋 <wf...@163.com>.
这个原因知道了 是因为hiveserver jvm 方法栈设置低，之前设置128k，但是hiveserver里udf 类的函数需要递归 等，需求较大，修正为512k，已经稳定运行3天。

At 2011-12-14 10:21:53,"王锋" <wf...@163.com> wrote:

when hiveserver is executing sql, it throws an exception:


2011-12-14 00:40:43,762 Stage-1 map = 90%,  reduce = 13%
2011-12-14 00:40:45,771 Stage-1 map = 92%,  reduce = 13%
2011-12-14 00:40:46,777 Stage-1 map = 96%,  reduce = 13%
2011-12-14 00:40:47,782 Stage-1 map = 97%,  reduce = 13%
2011-12-14 00:40:48,787 Stage-1 map = 98%,  reduce = 13%
2011-12-14 00:40:49,791 Stage-1 map = 99%,  reduce = 13%
2011-12-14 00:40:50,796 Stage-1 map = 100%,  reduce = 13%
2011-12-14 00:40:51,800 Stage-1 map = 100%,  reduce = 30%
2011-12-14 00:40:58,830 Stage-1 map = 100%,  reduce = 100%
Ended Job = job_201111141853_635398
OK
Exception in thread "pool-1-thread-100" java.lang.StackOverflowError
    at java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
    at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
 at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:356)
    at com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.findProviderClass(ObjectFactory.java:398)
    at com.sun.org.apache.xerces.internal.impl.dv.ObjectFactory.newInstance(ObjectFactory.java:353)
    at com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:62)
    at com.sun.org.apache.xerces.internal.impl.dv.DTDDVFactory.getInstance(DTDDVFactory.java:47)
    at com.sun.org.apache.xerces.internal.parsers.XML11Configuration.<init>(XML11Configuration.java:565)
    at com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:130)
    at com.sun.org.apache.xerces.internal.parsers.XIncludeAwareParserConfiguration.<init>(XIncludeAwareParserConfiguration.java:91)
    at sun.reflect.GeneratedConstructorAccessor2.newInstance(Unknown Source)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
    at java.lang.Class.newInstance0(Class.java:372)
    at java.lang.Class.newInstance(Class.java:325)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.newInstance(ObjectFactory.java:352)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:157)
    at com.sun.org.apache.xerces.internal.parsers.ObjectFactory.createObject(ObjectFactory.java:100)
    at com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:138)
    at com.sun.org.apache.xerces.internal.parsers.DOMParser.<init>(DOMParser.java:122)
    at com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderImpl.<init>(DocumentBuilderImpl.java:120)
    at com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl.newDocumentBuilder(DocumentBuilderFactoryImpl.java:76)
    at org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:1282)
    at org.apache.hadoop.conf.Configuration.loadResources(Configuration.java:1251)
    at org.apache.hadoop.conf.Configuration.getProps(Configuration.java:1192)
    at org.apache.hadoop.conf.Configuration.set(Configuration.java:461)
    at org.apache.hadoop.mapred.JobConf.setJar(JobConf.java:426)
    at org.apache.hadoop.mapred.JobConf.setJarByClass(JobConf.java:444)
    at org.apache.hadoop.mapred.JobConf.<init>(JobConf.java:339)
    at org.apache.hadoop.hive.conf.HiveConf.initialize(HiveConf.java:617)
    at org.apache.hadoop.hive.conf.HiveConf.<init>(HiveConf.java:588)
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:190)
    at org.apache.hadoop.hive.service.HiveServer$HiveServerHandler.<init>(HiveServer.java:94)
    at org.apache.hadoop.hive.service.HiveServer$ThriftHiveProcessorFactory.getProcessor(HiveServer.java:503)
    at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:246)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
    at java.lang.Thread.run(Thread.java:722)
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140042_2056220006.txt
37860.048: [GC 37860.048: [ParNew: 2529200K->3304K(3686400K), 0.0133750 secs] 2634845K->108963K(4710400K), 0.0134870 secs] [Times: user=0.06 sys=0.00, real=0.02 secs]
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140043_1286096583.txt
37920.093: [GC 37920.093: [ParNew: 1918585K->3218K(3686400K), 0.0117880 secs] 2024244K->108877K(4710400K), 0.0118950 secs] [Times: user=0.04 sys=0.00, real=0.01 secs]
Hive history file=/tmp/hdfs/hive_job_log_hdfs_201112140044_1787703059.txt
37980.126: [GC 37980.126: [ParNew: 1913955K->3238K(3686400K), 0.0128510 secs] 2019613K->108896K(4710400K), 0.0129470 secs] [Times: user=0.07 sys=0.00, real=0.02 secs]
Exception in thread "pool-1-thread-101" java.lang.StackOverflowError
    at java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1079)
    at java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:462)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:403)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:410)