You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "liuguanghua (JIRA)" <ji...@apache.org> on 2016/05/04 12:27:12 UTC
[jira] [Commented] (HIVE-13687) java.lang.RuntimeException: Reduce
operator initialization failed
[ https://issues.apache.org/jira/browse/HIVE-13687?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15270544#comment-15270544 ]
liuguanghua commented on HIVE-13687:
------------------------------------
when i run under the sql，it throw upper errors。
set spark.executor.instances=500;
add jar ./GroupConcat.jar;
create temporary function group_concat as 'com.letv.udaf.GroupConcat';
CREATE TEMPORARY FUNCTION MAXVALUE AS 'com.letv.udf.MaxValue';
select b.id,b.pid,b.vv,b.cv,c.gender,c.age,c.edu,c.purpower
from
(
select
id,
pid,
sum(case when (init+play+time) >0 then 1 else 0 end) vv,
sum(case when (play+time) > 0 then 1 else 0 end) cv
from data_sum.sum_user_uuid_play_day
where dt>20160318 and dt<=20160417 and pid!='-' and (case when (play+time) > 0 then 1 else 0 end)>0
group by id,pid
)b
join
(
select id,
MAXVALUE(gender,0.5) gender,
MAXVALUE(age,0) as age,
'-' as edu,
'-' as purpower
from temp.temp_predict_tags
where dt='20160417'
)c
on b.id=c.id;
> java.lang.RuntimeException: Reduce operator initialization failed
> -----------------------------------------------------------------
>
> Key: HIVE-13687
> URL: https://issues.apache.org/jira/browse/HIVE-13687
> Project: Hive
> Issue Type: Bug
> Affects Versions: 2.0.0
> Environment: hadoop2.6.4+hive2.0.0+spark1.6.1
> I run sql on the hive on spark model.
> Reporter: liuguanghua
>
> java.lang.RuntimeException: Reduce operator initialization failed
> at org.apache.hadoop.hive.ql.exec.spark.SparkReduceRecordHandler.init(SparkReduceRecordHandler.java:210)
> at org.apache.hadoop.hive.ql.exec.spark.HiveReduceFunction.call(HiveReduceFunction.java:46)
> at org.apache.hadoop.hive.ql.exec.spark.HiveReduceFunction.call(HiveReduceFunction.java:28)
> at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:192)
> at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$7$1.apply(JavaRDDLike.scala:192)
> at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$20.apply(RDD.scala:710)
> at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$20.apply(RDD.scala:710)
> at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:306)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:270)
> at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
> at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
> at org.apache.spark.scheduler.Task.run(Task.scala:89)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.IndexOutOfBoundsException: Index: 5, Size: 5
> at java.util.ArrayList.rangeCheck(ArrayList.java:635)
> at java.util.ArrayList.get(ArrayList.java:411)
> at org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.init(StandardStructObjectInspector.java:118)
> at org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector.<init>(StandardStructObjectInspector.java:109)
> at org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector(ObjectInspectorFactory.java:326)
> at org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector(ObjectInspectorFactory.java:311)
> at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.getJoinOutputObjectInspector(CommonJoinOperator.java:181)
> at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.initializeOp(CommonJoinOperator.java:319)
> at org.apache.hadoop.hive.ql.exec.JoinOperator.initializeOp(JoinOperator.java:70)
> at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:355)
> at org.apache.hadoop.hive.ql.exec.spark.SparkReduceRecordHandler.init(SparkReduceRecordHandler.java:195)
> ... 16 more
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)