You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "Henry Lu (Jira)" <ji...@apache.org> on 2020/07/14 09:25:00 UTC
[jira] [Updated] (HIVE-23844) use fastparquet generate parquet
format file, imported into hive, query error
[ https://issues.apache.org/jira/browse/HIVE-23844?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Henry Lu updated HIVE-23844:
----------------------------
Summary: use fastparquet generate parquet format file, imported into hive, query error (was: use fastparquet generate parquet format file, import to hive, query error)
> use fastparquet generate parquet format file, imported into hive, query error
> -------------------------------------------------------------------------------
>
> Key: HIVE-23844
> URL: https://issues.apache.org/jira/browse/HIVE-23844
> Project: Hive
> Issue Type: Bug
> Components: Hive
> Affects Versions: 2.1.1
> Environment: OS: CentOS Linux release 7.6.1810
> JDK: 1.8.0_181
> hive: hive-common-2.1.1-cdh6.2.0
> fastparquet: 0.4.0
> Reporter: Henry Lu
> Priority: Major
> Labels: fastparquet, hive, python
> Attachments: 1912076_20200330_000334(102_4).parquet
>
>
> i use fastparquet generate parquet format file(Please check the attachment ！), message as follows:
> message schema
> {
> optional double timestamps;
> optional int32 ESC_BrakePressure (UINT_8);
> optional int32 ESC_BrakePressureValid (UINT_8);
> optional int32 ESC_EBDWork (UINT_8);
> optional int32 ESC_ABSWorkLable (UINT_8);
> optional int32 ESC_EBDAlarm (UINT_8);
> optional int32 ESC_VehSpdValidFlag (UINT_8);
> optional int32 ESC_ABSAlarmSignal (UINT_8);
> optional float ESC_VehSpd;
> optional float ESC_FrontLeftWHeelSpd;
> optional int32 ESC_FLWHeelSpdFaultSignal (UINT_8);
> optional float ESC_FrontRightWHeelSpd;
> optional int32 ESC_FRWHeelSpdFaultSignal (UINT_8);
> optional float ESC_RearLeftWheelSpd;
> optional int32 ESC_RLWHeelSpdFaultSignal (UINT_8);
> optional float ESC_RearRightWheelSpd;
> optional int32 ESC_RRWHeelSpdFaultSignal (UINT_8);
> optional int32 ESC_Longitudinal_Acceleration_flag (UINT_8);
> optional float ESC_Longitudinal_Acceleration;
> optional int32 ESC_ESCOFF (UINT_8);
> optional int32 ESC_ESCWorkStatus (UINT_8);
> optional int32 ESC_ESCAlarmSig (UINT_8);
> optional int32 ESC_TCSCFActive (UINT_8);
> optional int32 ESC_ReqIncreaseTorqueFlag (UINT_8);
> optional int32 ESC_ReqDecreaseTorqueFlag (UINT_8);
> optional int32 ESC_ReqIncreaseTorque (UINT_8);
> optional int32 ESC_ReqDecreaseTorque (UINT_8);
> optional int32 ESC_ESCValidity (UINT_8);
> optional int32 ESC_RollingCount_ESC3 (UINT_8);
> optional int32 ESC_CICkSum_ESC3 (UINT_8);
> }
> creator: fastparquet-python version 1.0.0 (build 111)
> extra: pandas = {"column_indexes": [
> {"field_name": null, "metadata": null, "name": null, "numpy_type": "object", "pandas_type": "mixed-integer"}
> ], "columns": [
> {"field_name": "timestamps", "metadata": null, "name": "timestamps", "numpy_type": "float64", "pandas_type": "float64"}
> ,
> {"field_name": "ESC_BrakePressure", "metadata": null, "name": "ESC_BrakePressure", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_BrakePressureValid", "metadata": null, "name": "ESC_BrakePressureValid", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_EBDWork", "metadata": null, "name": "ESC_EBDWork", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ABSWorkLable", "metadata": null, "name": "ESC_ABSWorkLable", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_EBDAlarm", "metadata": null, "name": "ESC_EBDAlarm", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_VehSpdValidFlag", "metadata": null, "name": "ESC_VehSpdValidFlag", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ABSAlarmSignal", "metadata": null, "name": "ESC_ABSAlarmSignal", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_VehSpd", "metadata": null, "name": "ESC_VehSpd", "numpy_type": "float32", "pandas_type": "float32"}
> ,
> {"field_name": "ESC_FrontLeftWHeelSpd", "metadata": null, "name": "ESC_FrontLeftWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}
> ,
> {"field_name": "ESC_FLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_FrontRightWHeelSpd", "metadata": null, "name": "ESC_FrontRightWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}
> ,
> {"field_name": "ESC_FRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_RearLeftWheelSpd", "metadata": null, "name": "ESC_RearLeftWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}
> ,
> {"field_name": "ESC_RLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_RearRightWheelSpd", "metadata": null, "name": "ESC_RearRightWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}
> ,
> {"field_name": "ESC_RRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_Longitudinal_Acceleration_flag", "metadata": null, "name": "ESC_Longitudinal_Acceleration_flag", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_Longitudinal_Acceleration", "metadata": null, "name": "ESC_Longitudinal_Acceleration", "numpy_type": "float32", "pandas_type": "float32"}
> ,
> {"field_name": "ESC_ESCOFF", "metadata": null, "name": "ESC_ESCOFF", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ESCWorkStatus", "metadata": null, "name": "ESC_ESCWorkStatus", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ESCAlarmSig", "metadata": null, "name": "ESC_ESCAlarmSig", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_TCSCFActive", "metadata": null, "name": "ESC_TCSCFActive", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ReqIncreaseTorqueFlag", "metadata": null, "name": "ESC_ReqIncreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ReqDecreaseTorqueFlag", "metadata": null, "name": "ESC_ReqDecreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ReqIncreaseTorque", "metadata": null, "name": "ESC_ReqIncreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ReqDecreaseTorque", "metadata": null, "name": "ESC_ReqDecreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_ESCValidity", "metadata": null, "name": "ESC_ESCValidity", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_RollingCount_ESC3", "metadata": null, "name": "ESC_RollingCount_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}
> ,
> {"field_name": "ESC_CICkSum_ESC3", "metadata": null, "name": "ESC_CICkSum_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}
> ], "creator": \{"library": "fastparquet", "version": "0.4.0"}, "index_columns": ["timestamps"], "pandas_version": "1.0.1"}
> file schema: schema
> --------------------------------------------------------------------------------
> timestamps: OPTIONAL DOUBLE R:0 D:1
> ESC_BrakePressure: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_BrakePressureValid: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_EBDWork: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ABSWorkLable: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_EBDAlarm: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_VehSpdValidFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ABSAlarmSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_VehSpd: OPTIONAL FLOAT R:0 D:1
> ESC_FrontLeftWHeelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_FLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_FrontRightWHeelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_FRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_RearLeftWheelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_RLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_RearRightWheelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_RRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_Longitudinal_Acceleration_flag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_Longitudinal_Acceleration: OPTIONAL FLOAT R:0 D:1
> ESC_ESCOFF: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ESCWorkStatus: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ESCAlarmSig: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_TCSCFActive: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqIncreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqDecreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqIncreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqDecreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ESCValidity: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_RollingCount_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_CICkSum_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
> row group 1: RC:67096 TS:8321051 OFFSET:4
> --------------------------------------------------------------------------------
> timestamps: DOUBLE GZIP DO:0 FPO:4 SZ:168467/536807/3.19 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_BrakePressure: INT32 GZIP DO:0 FPO:168549 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_BrakePressureValid: INT32 GZIP DO:0 FPO:168954 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_EBDWork: INT32 GZIP DO:0 FPO:169364 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ABSWorkLable: INT32 GZIP DO:0 FPO:169763 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_EBDAlarm: INT32 GZIP DO:0 FPO:170167 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_VehSpdValidFlag: INT32 GZIP DO:0 FPO:170567 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ABSAlarmSignal: INT32 GZIP DO:0 FPO:170974 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_VehSpd: FLOAT GZIP DO:0 FPO:171380 SZ:118126/268423/2.27 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FrontLeftWHeelSpd: FLOAT GZIP DO:0 FPO:289578 SZ:111048/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:400709 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FrontRightWHeelSpd: FLOAT GZIP DO:0 FPO:401122 SZ:110788/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:511994 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RearLeftWheelSpd: FLOAT GZIP DO:0 FPO:512407 SZ:114567/268423/2.34 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:627056 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RearRightWheelSpd: FLOAT GZIP DO:0 FPO:627469 SZ:115113/268423/2.33 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:742665 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_Longitudinal_Acceleration_flag: INT32 GZIP DO:0 FPO:743078 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_Longitudinal_Acceleration: FLOAT GZIP DO:0 FPO:743500 SZ:46863/268423/5.73 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCOFF: INT32 GZIP DO:0 FPO:790454 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCWorkStatus: INT32 GZIP DO:0 FPO:790852 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCAlarmSig: INT32 GZIP DO:0 FPO:791257 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_TCSCFActive: INT32 GZIP DO:0 FPO:791660 SZ:332/268422/808.50 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqIncreaseTorqueFlag: INT32 GZIP DO:0 FPO:792068 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqDecreaseTorqueFlag: INT32 GZIP DO:0 FPO:792481 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqIncreaseTorque: INT32 GZIP DO:0 FPO:792894 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqDecreaseTorque: INT32 GZIP DO:0 FPO:793303 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCValidity: INT32 GZIP DO:0 FPO:793712 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RollingCount_ESC3: INT32 GZIP DO:0 FPO:794115 SZ:4339/268422/61.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_CICkSum_ESC3: INT32 GZIP DO:0 FPO:798536 SZ:4341/268422/61.83 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
>
> hive create table sql:
> create external table if not exists test(`timestamps` double, `ESC_BrakePressure` int, `ESC_EBDWork` int)
> stored as parquet
> location '/dfmc/testcar'
> TBLPROPERTIES ('PARQUET.COMPRESS'='GZIP');
>
> load data inpath '/dfmc/testcar' into table test;
>
> exe sql : select * from test;
>
> error:
> Failed with exception java.io.IOException:java.lang.IllegalStateException: Group type [message schema
> { optional double timestamps; optional int32 ESC_BrakePressure (INTEGER(8,false)); optional int32 ESC_EBDWork (INTEGER(8,false)); }
> ] does not contain requested field: optional double timestamps
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)