You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "ASF GitHub Bot (Jira)" <ji...@apache.org> on 2026/09/17 10:53:00 UTC
[jira] [Updated] (HIVE-30043) FULL OUTER MapJoin (dynamic partitioned hash join) throws ClassCastException: LongWritable cannot be cast to ShortWritable
[ https://issues.apache.org/jira/browse/HIVE-30043?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
ASF GitHub Bot updated HIVE-30043:
----------------------------------
Labels: pull-request-available (was: )
> FULL OUTER MapJoin (dynamic partitioned hash join) throws ClassCastException: LongWritable cannot be cast to ShortWritable
> --------------------------------------------------------------------------------------------------------------------------
>
> Key: HIVE-30043
> URL: https://issues.apache.org/jira/browse/HIVE-30043
> Project: Hive
> Issue Type: Bug
> Reporter: Indhumathi Muthumurugesh
> Assignee: Indhumathi Muthumurugesh
> Priority: Major
> Labels: pull-request-available
>
> With {{hive.optimize.dynamic.partition.hashjoin=true}} and a query shape that forces row-mode (non-vectorized) execution of a {{MapJoinOperator}} implementing a {{{}FULL OUTER JOIN{}}}, the query fails with:
> {code:java}
> java.lang.ClassCastException: class org.apache.hadoop.io.LongWritable cannot be cast to
> class org.apache.hadoop.hive.serde2.io.ShortWritable {code}
> The crash lands on a BIGINT column (derived from count(1)) that is being misinterpreted as SMALLINT. The same query, same data, and same settings run correctly when the plan is fully vectorized instead (hive.vectorized.adaptor.usage.mode=all), or when the dynamic-partitioned-hash-join conversion is disabled.
> h2. Stack Trace
> {code:java}
> ERROR : FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask.Vertex failed, vertexName=Reducer 8, ...Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:313) at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:291) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:293) ... 16 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:387) at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:303) ... 18 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unexpected exception from MapJoinOperator : Error evaluating if(_col22 is not null, _col22, 0L) at org.apache.hadoop.hive.ql.exec.MapJoinOperator.process(MapJoinOperator.java:599) at org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:372) ... 19 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Error evaluating if(_col22 is not null, _col22, 0L) at org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:92) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:888) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.internalForward(CommonJoinOperator.java:889) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:921) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:1015) at org.apache.hadoop.hive.ql.exec.MapJoinOperator.process(MapJoinOperator.java:586) ... 20 moreCaused by: java.lang.ClassCastException: class org.apache.hadoop.io.LongWritable cannot be cast to class org.apache.hadoop.hive.serde2.io.ShortWritable (org.apache.hadoop.io.LongWritable and org.apache.hadoop.hive.serde2.io.ShortWritable are in unnamed module of loader 'app') at org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector.get(WritableShortObjectInspector.java:36) at org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.getLong(PrimitiveObjectInspectorUtils.java:777) at org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter$LongConverter.convert(PrimitiveObjectInspectorConverter.java:183) at org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils$ReturnObjectInspectorResolver.convertIfNecessary(GenericUDFUtils.java:247) at org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils$ReturnObjectInspectorResolver.convertIfNecessary(GenericUDFUtils.java:213) at org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf.evaluate(GenericUDFIf.java:168) at org.apache.hadoop.hive.ql.exec.ExprNodeGenericFuncEvaluator._evaluate(ExprNodeGenericFuncEvaluator.java:231) at org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:80) at org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorHead._evaluate(ExprNodeEvaluatorHead.java:44) at org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:80) at org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator.evaluate(ExprNodeEvaluator.java:68) at org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:87) ... 25 more {code}
> h2. Steps to Reproduce
> {code:java}
> set hive.execution.engine=tez;
> set hive.auto.convert.join=true;
> set hive.optimize.dynamic.partition.hashjoin=true;
> set hive.vectorized.adaptor.usage.mode=chosen;DROP TABLE IF EXISTS t_base;
> CREATE TABLE t_base (
> entity_id STRING,
> proc_type STRING,
> event_dt STRING,
> code_list STRING,
> notif_cd STRING,
> category_cd STRING,
> status_cd STRING,
> amount_usd DOUBLE,
> flag_ind STRING,
> flag_cnt SMALLINT
> ) STORED AS ORC;INSERT INTO TABLE t_base VALUES
> ('BIN001','EDIT','20260101','045','1','0','C',100.00,'N',CAST(0 AS SMALLINT)),
> ('BIN001','EDIT','20260101','045','1','1','C',200.00,'N',CAST(0 AS SMALLINT)),
> ('BIN001','EDIT','20260101','000','3','2','W',50.00,'N',CAST(0 AS SMALLINT)),
> ('BIN001','EDIT','20260101','000','1','0','A',75.00,'N',CAST(0 AS SMALLINT)),
> ('BIN001','EDIT','20260101','000','1','3','S',10.00,'N',CAST(0 AS SMALLINT)),
> ('BIN002','EDIT','20260101','000','1','1','C',300.00,'N',CAST(0 AS SMALLINT)),
> ('BIN002','EDIT','20260101','000','1','0','U',20.00,'N',CAST(0 AS SMALLINT)),
> ('BIN002','EDIT','20260101','000','1','4','T',5.00,'N',CAST(0 AS SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','5','C',400.00,'Y',CAST(1 AS SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','6','C',150.00,'Y',CAST(1 AS SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','0','A',60.00,'N',CAST(0 AS SMALLINT)),
> ('BIN003','EDIT','20260101','000','1','1','W',30.00,'N',CAST(0 AS SMALLINT));CREATE TEMPORARY FUNCTION test_uuid AS 'org.apache.hadoop.hive.ql.udf.UDFUUID';SELECT coalesce(join_a.entity_id, join_b.entity_id) as entity_id,
> coalesce(join_a.warn_code_1, '000') as warn_code_1,
> coalesce(join_a.warn_1_total, 0) as warn_1_total,
> coalesce(join_a.notif_1_total, 0) as notif_1_total,
> coalesce(join_b.cat_0_tran_count, 0) as cat_0_tran_count,
> coalesce(if(join_b.cat_0_dollar_total > 9999999999.99, 9999999999.99, join_b.cat_0_dollar_total), 0) as cat_0_dollar_total,
> COALESCE(join_b.flag_cnt, 0) as flag_cnt,
> test_uuid() as row_id
> FROM
> (select warn_agg.entity_id,
> warn_agg.warn_code_1,
> warn_agg.warn_1_total,
> notif_agg.notif_1_total
> from
> (Select d.entity_id,
> COALESCE(d.code_list[0], '000') as warn_code_1,
> cast(COALESCE(d.count_list[0], 0) as int) as warn_1_total
> from
> (SELECT c.entity_id, collect_list(c.code) as code_list, collect_list(c.count) as count_list
> FROM
> (select b.entity_id, b.code, count(1) as count
> from (
> select entity_id, trim(code) as code
> from (
> select entity_id, split(CONCAT(SUBSTR(RPAD(TRIM(regexp_replace(code_list,'=','0')),6,'0'),0,3),'-',SUBSTR(RPAD(TRIM(regexp_replace(code_list,'=','0')),6,'0'),4,3)), '-') as code_array
> from t_base where proc_type = 'EDIT' and event_dt in ('20260101')) a
> lateral view explode(a.code_array) exploded as code) b
> where b.code != '000'
> group by b.entity_id, b.code) c
> GROUP BY c.entity_id order by c.entity_id) d) warn_agg
> join
> (Select e.entity_id,
> cast(COALESCE(e.count_list[0], 0) as int) as notif_1_total
> from
> (SELECT d.entity_id, collect_list(d.notif_cd) as notif_cd_list, collect_list(d.count) as count_list
> FROM
> (select b.entity_id, b.notif_cd, COALESCE(a.count, 0) as count
> from
> (select entity_id, trim(notif_cd) as notif_cd
> from (
> select distinct entity_id, split(CONCAT(SUBSTR('13',1,1),'-',SUBSTR('13',2,1)), '-') as notif_cd_array
> from t_base where proc_type = 'EDIT' and event_dt in ('20260101')) a
> lateral view explode(a.notif_cd_array) exploded as notif_cd) b
> left outer join
> (SELECT c.entity_id, c.notif_cd, count(1) as count
> FROM (select entity_id, notif_cd from t_base
> where proc_type = 'EDIT' and event_dt in ('20260101')) c
> group by c.entity_id, c.notif_cd order by c.notif_cd) a
> on a.entity_id = b.entity_id and a.notif_cd = b.notif_cd
> order by b.entity_id, b.notif_cd) d
> GROUP BY d.entity_id) e) notif_agg
> on warn_agg.entity_id = notif_agg.entity_id) join_a
> full outer join
> (Select d.entity_id,
> cast(d.count_list[0] as int) as cat_0_tran_count,
> cast(d.amount_usd_list[0] as double) as cat_0_dollar_total,
> flag_agg.flag_cnt
> FROM
> (SELECT c.entity_id, collect_list(c.category_code) as category_code_list, collect_list(c.count) as count_list,
> collect_list(cast(c.amount_usd as string)) as amount_usd_list
> FROM
> (select b.entity_id, b.category_code, COALESCE(a.count, 0) as count, COALESCE(a.amount_usd, 0.0) as amount_usd
> from
> (select entity_id, trim(category_code) as category_code
> from (
> select distinct entity_id, split(CONCAT(SUBSTR('01',1,1),'-',SUBSTR('01',2,1)), '-') as category_code_array
> from t_base where proc_type = 'EDIT' and event_dt in ('20260101')) a
> lateral view explode(a.category_code_array) exploded as category_code) b
> left outer join
> (select entity_id, category_cd, count(1) as count,
> sum(case when status_cd in ('C','W','A','S','U','T') then amount_usd else 0 end) as amount_usd
> from t_base where proc_type = 'EDIT' and event_dt in ('20260101')
> group by entity_id, category_cd order by category_cd) a
> on a.entity_id = b.entity_id and a.category_cd = b.category_code
> order by b.entity_id, b.category_code) c
> GROUP BY c.entity_id) d
> left outer join
> (select entity_id, count(1) as flag_cnt from t_base
> where proc_type = 'EDIT' and event_dt in ('20260101') and flag_ind = 'Y'
> group by entity_id) flag_agg
> on d.entity_id = flag_agg.entity_id
> ) join_b
> on join_a.entity_id = join_b.entity_id; {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)