You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "翟玉勇 (JIRA)" <ji...@apache.org> on 2018/03/14 08:00:00 UTC
[jira] [Updated] (HIVE-18954) insert hive storage handler table
from a parquet table error
[ https://issues.apache.org/jira/browse/HIVE-18954?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
翟玉勇 updated HIVE-18954:
-----------------------
Description:
{code:java}
drop table if exists temp.temp_dw_sia_jc_kpa7_brand_factor_list_week;
create table temp.temp_dw_sia_jc_kpa7_brand_factor_list_week(
`brand_id` bigint,
`brand_name` string,
`reason_key` string,
`reason_value` double ,
`reason_imp_order_num` double ,
`data_type` string ,
`biz_date` string
)
STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
TBLPROPERTIES (
'es.resource'='index/type',
'es.nodes'='',
'es.port'='9200',
'es.nodes.wan.only'='true',
'es.index.translog.flush_threshold_size'='3gb',
'es.index.translog.interval'='60s',
'es.index.translog,sync_interval'='60s',
'index.translog.durability'='async'
);
insert overwrite table temp.temp_dw_sia_jc_kpa7_brand_factor_list
select
now_brand_id as brand_id,
now_brand_name as brand_name,
factor as reason_key,
value as reason_value,
value_order as reason_imp_order_num,
model as data_type,
biz_date
from dw.dw_jc_kpa7_brand_factor_list
where dt = '123'
and value is not null and value_order is not null and model is not null;
{code}
dw.dw_jc_kpa7_brand_factor_list is a paruet table
{code}
2018-03-14 05:05:50,346 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : java.io.IOException: java.lang.reflect.InvocationTargetException
at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97)
at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57)
at org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.initNextRecordReader(HadoopShimsSecure.java:269)
at org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.next(HadoopShimsSecure.java:143)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.moveToNext(MapTask.java:199)
at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.next(MapTask.java:185)
at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:52)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:453)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1693)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.initNextRecordReader(HadoopShimsSecure.java:255)
... 11 more
Caused by: java.lang.IndexOutOfBoundsException: Index: 7, Size: 7
at java.util.ArrayList.rangeCheck(ArrayList.java:635)
at java.util.ArrayList.get(ArrayList.java:411)
at org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport.getProjectedGroupFields(DataWritableReadSupport.java:116)
at org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport.getSchemaByName(DataWritableReadSupport.java:176)
at org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport.init(DataWritableReadSupport.java:242)
at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.getSplit(ParquetRecordReaderWrapper.java:248)
at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.<init>(ParquetRecordReaderWrapper.java:94)
at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.<init>(ParquetRecordReaderWrapper.java:80)
at org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat.getRecordReader(MapredParquetInputFormat.java:72)
at org.apache.hadoop.hive.ql.io.CombineHiveRecordReader.<init>(CombineHiveRecordReader.java:67)
... 16 more
{code}
was:
{code}
drop table if exists temp.temp_dw_sia_jc_kpa7_brand_factor_list_week;
create table temp.temp_dw_sia_jc_kpa7_brand_factor_list_week(
`brand_id` bigint,
`brand_name` string,
`reason_key` string,
`reason_value` double ,
`reason_imp_order_num` double ,
`data_type` string ,
`biz_date` string
)
STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
TBLPROPERTIES (
'es.resource'='index/type',
'es.nodes'='',
'es.port'='9200',
'es.nodes.wan.only'='true',
'es.index.translog.flush_threshold_size'='3gb',
'es.index.translog.interval'='60s',
'es.index.translog,sync_interval'='60s',
'index.translog.durability'='async'
);
insert overwrite table temp.temp_dw_sia_jc_kpa7_brand_factor_list
select
now_brand_id as brand_id,
now_brand_name as brand_name,
factor as reason_key,
value as reason_value,
value_order as reason_imp_order_num,
model as data_type,
biz_date
from dw.dw_jc_kpa7_brand_factor_list
where dt = '123'
and value is not null and value_order is not null and model is not null;
{code}
> insert hive storage handler table from a parquet table error
> ------------------------------------------------------------
>
> Key: HIVE-18954
> URL: https://issues.apache.org/jira/browse/HIVE-18954
> Project: Hive
> Issue Type: Bug
> Components: StorageHandler
> Affects Versions: 2.1.0
> Reporter: 翟玉勇
> Priority: Blocker
>
> {code:java}
> drop table if exists temp.temp_dw_sia_jc_kpa7_brand_factor_list_week;
> create table temp.temp_dw_sia_jc_kpa7_brand_factor_list_week(
> `brand_id` bigint,
> `brand_name` string,
> `reason_key` string,
> `reason_value` double ,
> `reason_imp_order_num` double ,
> `data_type` string ,
> `biz_date` string
> )
> STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler'
> TBLPROPERTIES (
> 'es.resource'='index/type',
> 'es.nodes'='',
> 'es.port'='9200',
> 'es.nodes.wan.only'='true',
> 'es.index.translog.flush_threshold_size'='3gb',
> 'es.index.translog.interval'='60s',
> 'es.index.translog,sync_interval'='60s',
> 'index.translog.durability'='async'
> );
> insert overwrite table temp.temp_dw_sia_jc_kpa7_brand_factor_list
> select
> now_brand_id as brand_id,
> now_brand_name as brand_name,
> factor as reason_key,
> value as reason_value,
> value_order as reason_imp_order_num,
> model as data_type,
> biz_date
> from dw.dw_jc_kpa7_brand_factor_list
> where dt = '123'
> and value is not null and value_order is not null and model is not null;
> {code}
> dw.dw_jc_kpa7_brand_factor_list is a paruet table
> {code}
> 2018-03-14 05:05:50,346 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : java.io.IOException: java.lang.reflect.InvocationTargetException
> at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97)
> at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57)
> at org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.initNextRecordReader(HadoopShimsSecure.java:269)
> at org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.next(HadoopShimsSecure.java:143)
> at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.moveToNext(MapTask.java:199)
> at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.next(MapTask.java:185)
> at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:52)
> at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:453)
> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
> at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:415)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1693)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
> Caused by: java.lang.reflect.InvocationTargetException
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
> at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
> at org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileRecordReader.initNextRecordReader(HadoopShimsSecure.java:255)
> ... 11 more
> Caused by: java.lang.IndexOutOfBoundsException: Index: 7, Size: 7
> at java.util.ArrayList.rangeCheck(ArrayList.java:635)
> at java.util.ArrayList.get(ArrayList.java:411)
> at org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport.getProjectedGroupFields(DataWritableReadSupport.java:116)
> at org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport.getSchemaByName(DataWritableReadSupport.java:176)
> at org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport.init(DataWritableReadSupport.java:242)
> at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.getSplit(ParquetRecordReaderWrapper.java:248)
> at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.<init>(ParquetRecordReaderWrapper.java:94)
> at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.<init>(ParquetRecordReaderWrapper.java:80)
> at org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat.getRecordReader(MapredParquetInputFormat.java:72)
> at org.apache.hadoop.hive.ql.io.CombineHiveRecordReader.<init>(CombineHiveRecordReader.java:67)
> ... 16 more
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)