You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hive.apache.org by "Henry Lu (Jira)" <ji...@apache.org> on 2020/07/14 08:59:00 UTC
[jira] [Updated] (HIVE-23844) use fastparquet generate parquet
format file, import hive, query error
[ https://issues.apache.org/jira/browse/HIVE-23844?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Henry Lu updated HIVE-23844:
----------------------------
Description:
i use fastparquet generate parquet format file(Please check the attachment ！), message as follows:
message schema
{
optional double timestamps;
optional int32 ESC_BrakePressure (UINT_8);
optional int32 ESC_BrakePressureValid (UINT_8);
optional int32 ESC_EBDWork (UINT_8);
optional int32 ESC_ABSWorkLable (UINT_8);
optional int32 ESC_EBDAlarm (UINT_8);
optional int32 ESC_VehSpdValidFlag (UINT_8);
optional int32 ESC_ABSAlarmSignal (UINT_8);
optional float ESC_VehSpd;
optional float ESC_FrontLeftWHeelSpd;
optional int32 ESC_FLWHeelSpdFaultSignal (UINT_8);
optional float ESC_FrontRightWHeelSpd;
optional int32 ESC_FRWHeelSpdFaultSignal (UINT_8);
optional float ESC_RearLeftWheelSpd;
optional int32 ESC_RLWHeelSpdFaultSignal (UINT_8);
optional float ESC_RearRightWheelSpd;
optional int32 ESC_RRWHeelSpdFaultSignal (UINT_8);
optional int32 ESC_Longitudinal_Acceleration_flag (UINT_8);
optional float ESC_Longitudinal_Acceleration;
optional int32 ESC_ESCOFF (UINT_8);
optional int32 ESC_ESCWorkStatus (UINT_8);
optional int32 ESC_ESCAlarmSig (UINT_8);
optional int32 ESC_TCSCFActive (UINT_8);
optional int32 ESC_ReqIncreaseTorqueFlag (UINT_8);
optional int32 ESC_ReqDecreaseTorqueFlag (UINT_8);
optional int32 ESC_ReqIncreaseTorque (UINT_8);
optional int32 ESC_ReqDecreaseTorque (UINT_8);
optional int32 ESC_ESCValidity (UINT_8);
optional int32 ESC_RollingCount_ESC3 (UINT_8);
optional int32 ESC_CICkSum_ESC3 (UINT_8);
}
creator: fastparquet-python version 1.0.0 (build 111)
extra: pandas = {"column_indexes": [
{"field_name": null, "metadata": null, "name": null, "numpy_type": "object", "pandas_type": "mixed-integer"}
], "columns": [\\{"field_name": "timestamps", "metadata": null, "name": "timestamps", "numpy_type": "float64", "pandas_type": "float64"}, \\{"field_name": "ESC_BrakePressure", "metadata": null, "name": "ESC_BrakePressure", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_BrakePressureValid", "metadata": null, "name": "ESC_BrakePressureValid", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_EBDWork", "metadata": null, "name": "ESC_EBDWork", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ABSWorkLable", "metadata": null, "name": "ESC_ABSWorkLable", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_EBDAlarm", "metadata": null, "name": "ESC_EBDAlarm", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_VehSpdValidFlag", "metadata": null, "name": "ESC_VehSpdValidFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ABSAlarmSignal", "metadata": null, "name": "ESC_ABSAlarmSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_VehSpd", "metadata": null, "name": "ESC_VehSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_FrontLeftWHeelSpd", "metadata": null, "name": "ESC_FrontLeftWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_FLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_FrontRightWHeelSpd", "metadata": null, "name": "ESC_FrontRightWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_FRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_RearLeftWheelSpd", "metadata": null, "name": "ESC_RearLeftWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_RLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_RearRightWheelSpd", "metadata": null, "name": "ESC_RearRightWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_RRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_Longitudinal_Acceleration_flag", "metadata": null, "name": "ESC_Longitudinal_Acceleration_flag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_Longitudinal_Acceleration", "metadata": null, "name": "ESC_Longitudinal_Acceleration", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_ESCOFF", "metadata": null, "name": "ESC_ESCOFF", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ESCWorkStatus", "metadata": null, "name": "ESC_ESCWorkStatus", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ESCAlarmSig", "metadata": null, "name": "ESC_ESCAlarmSig", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_TCSCFActive", "metadata": null, "name": "ESC_TCSCFActive", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqIncreaseTorqueFlag", "metadata": null, "name": "ESC_ReqIncreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqDecreaseTorqueFlag", "metadata": null, "name": "ESC_ReqDecreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqIncreaseTorque", "metadata": null, "name": "ESC_ReqIncreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqDecreaseTorque", "metadata": null, "name": "ESC_ReqDecreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ESCValidity", "metadata": null, "name": "ESC_ESCValidity", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_RollingCount_ESC3", "metadata": null, "name": "ESC_RollingCount_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_CICkSum_ESC3", "metadata": null, "name": "ESC_CICkSum_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}], "creator": \{"library": "fastparquet", "version": "0.4.0"}, "index_columns": ["timestamps"], "pandas_version": "1.0.1"}
file schema: schema
--------------------------------------------------------------------------------
timestamps: OPTIONAL DOUBLE R:0 D:1
ESC_BrakePressure: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_BrakePressureValid: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_EBDWork: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ABSWorkLable: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_EBDAlarm: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_VehSpdValidFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ABSAlarmSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_VehSpd: OPTIONAL FLOAT R:0 D:1
ESC_FrontLeftWHeelSpd: OPTIONAL FLOAT R:0 D:1
ESC_FLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_FrontRightWHeelSpd: OPTIONAL FLOAT R:0 D:1
ESC_FRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RearLeftWheelSpd: OPTIONAL FLOAT R:0 D:1
ESC_RLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RearRightWheelSpd: OPTIONAL FLOAT R:0 D:1
ESC_RRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_Longitudinal_Acceleration_flag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_Longitudinal_Acceleration: OPTIONAL FLOAT R:0 D:1
ESC_ESCOFF: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCWorkStatus: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCAlarmSig: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_TCSCFActive: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqIncreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqDecreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqIncreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqDecreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCValidity: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RollingCount_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_CICkSum_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
row group 1: RC:67096 TS:8321051 OFFSET:4
--------------------------------------------------------------------------------
timestamps: DOUBLE GZIP DO:0 FPO:4 SZ:168467/536807/3.19 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_BrakePressure: INT32 GZIP DO:0 FPO:168549 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_BrakePressureValid: INT32 GZIP DO:0 FPO:168954 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_EBDWork: INT32 GZIP DO:0 FPO:169364 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ABSWorkLable: INT32 GZIP DO:0 FPO:169763 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_EBDAlarm: INT32 GZIP DO:0 FPO:170167 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_VehSpdValidFlag: INT32 GZIP DO:0 FPO:170567 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ABSAlarmSignal: INT32 GZIP DO:0 FPO:170974 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_VehSpd: FLOAT GZIP DO:0 FPO:171380 SZ:118126/268423/2.27 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FrontLeftWHeelSpd: FLOAT GZIP DO:0 FPO:289578 SZ:111048/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:400709 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FrontRightWHeelSpd: FLOAT GZIP DO:0 FPO:401122 SZ:110788/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:511994 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RearLeftWheelSpd: FLOAT GZIP DO:0 FPO:512407 SZ:114567/268423/2.34 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:627056 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RearRightWheelSpd: FLOAT GZIP DO:0 FPO:627469 SZ:115113/268423/2.33 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:742665 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_Longitudinal_Acceleration_flag: INT32 GZIP DO:0 FPO:743078 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_Longitudinal_Acceleration: FLOAT GZIP DO:0 FPO:743500 SZ:46863/268423/5.73 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCOFF: INT32 GZIP DO:0 FPO:790454 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCWorkStatus: INT32 GZIP DO:0 FPO:790852 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCAlarmSig: INT32 GZIP DO:0 FPO:791257 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_TCSCFActive: INT32 GZIP DO:0 FPO:791660 SZ:332/268422/808.50 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqIncreaseTorqueFlag: INT32 GZIP DO:0 FPO:792068 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqDecreaseTorqueFlag: INT32 GZIP DO:0 FPO:792481 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqIncreaseTorque: INT32 GZIP DO:0 FPO:792894 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqDecreaseTorque: INT32 GZIP DO:0 FPO:793303 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCValidity: INT32 GZIP DO:0 FPO:793712 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RollingCount_ESC3: INT32 GZIP DO:0 FPO:794115 SZ:4339/268422/61.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_CICkSum_ESC3: INT32 GZIP DO:0 FPO:798536 SZ:4341/268422/61.83 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
hive create table sql:
create external table if not exists test(`timestamps` double, `ESC_BrakePressure` int, `ESC_EBDWork` int)
stored as parquet
location '/dfmc/testcar'
TBLPROPERTIES ('PARQUET.COMPRESS'='GZIP');
load data inpath '/dfmc/testcar' into table test;
exe sql : select * from test;
error:
Failed with exception java.io.IOException:java.lang.IllegalStateException: Group type [message schema
{ optional double timestamps; optional int32 ESC_BrakePressure (INTEGER(8,false)); optional int32 ESC_EBDWork (INTEGER(8,false)); }
] does not contain requested field: optional double timestamps
was:
i use fastparquet generate parquet format file(Please check the attachment ！), message as follows:
message schema {
optional double timestamps;
optional int32 ESC_BrakePressure (UINT_8);
optional int32 ESC_BrakePressureValid (UINT_8);
optional int32 ESC_EBDWork (UINT_8);
optional int32 ESC_ABSWorkLable (UINT_8);
optional int32 ESC_EBDAlarm (UINT_8);
optional int32 ESC_VehSpdValidFlag (UINT_8);
optional int32 ESC_ABSAlarmSignal (UINT_8);
optional float ESC_VehSpd;
optional float ESC_FrontLeftWHeelSpd;
optional int32 ESC_FLWHeelSpdFaultSignal (UINT_8);
optional float ESC_FrontRightWHeelSpd;
optional int32 ESC_FRWHeelSpdFaultSignal (UINT_8);
optional float ESC_RearLeftWheelSpd;
optional int32 ESC_RLWHeelSpdFaultSignal (UINT_8);
optional float ESC_RearRightWheelSpd;
optional int32 ESC_RRWHeelSpdFaultSignal (UINT_8);
optional int32 ESC_Longitudinal_Acceleration_flag (UINT_8);
optional float ESC_Longitudinal_Acceleration;
optional int32 ESC_ESCOFF (UINT_8);
optional int32 ESC_ESCWorkStatus (UINT_8);
optional int32 ESC_ESCAlarmSig (UINT_8);
optional int32 ESC_TCSCFActive (UINT_8);
optional int32 ESC_ReqIncreaseTorqueFlag (UINT_8);
optional int32 ESC_ReqDecreaseTorqueFlag (UINT_8);
optional int32 ESC_ReqIncreaseTorque (UINT_8);
optional int32 ESC_ReqDecreaseTorque (UINT_8);
optional int32 ESC_ESCValidity (UINT_8);
optional int32 ESC_RollingCount_ESC3 (UINT_8);
optional int32 ESC_CICkSum_ESC3 (UINT_8);
}
creator: fastparquet-python version 1.0.0 (build 111)
extra: pandas = \{"column_indexes": [{"field_name": null, "metadata": null, "name": null, "numpy_type": "object", "pandas_type": "mixed-integer"}], "columns": [\{"field_name": "timestamps", "metadata": null, "name": "timestamps", "numpy_type": "float64", "pandas_type": "float64"}, \{"field_name": "ESC_BrakePressure", "metadata": null, "name": "ESC_BrakePressure", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_BrakePressureValid", "metadata": null, "name": "ESC_BrakePressureValid", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_EBDWork", "metadata": null, "name": "ESC_EBDWork", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ABSWorkLable", "metadata": null, "name": "ESC_ABSWorkLable", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_EBDAlarm", "metadata": null, "name": "ESC_EBDAlarm", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_VehSpdValidFlag", "metadata": null, "name": "ESC_VehSpdValidFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ABSAlarmSignal", "metadata": null, "name": "ESC_ABSAlarmSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_VehSpd", "metadata": null, "name": "ESC_VehSpd", "numpy_type": "float32", "pandas_type": "float32"}, \{"field_name": "ESC_FrontLeftWHeelSpd", "metadata": null, "name": "ESC_FrontLeftWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \{"field_name": "ESC_FLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_FrontRightWHeelSpd", "metadata": null, "name": "ESC_FrontRightWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \{"field_name": "ESC_FRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_RearLeftWheelSpd", "metadata": null, "name": "ESC_RearLeftWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \{"field_name": "ESC_RLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_RearRightWheelSpd", "metadata": null, "name": "ESC_RearRightWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \{"field_name": "ESC_RRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_Longitudinal_Acceleration_flag", "metadata": null, "name": "ESC_Longitudinal_Acceleration_flag", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_Longitudinal_Acceleration", "metadata": null, "name": "ESC_Longitudinal_Acceleration", "numpy_type": "float32", "pandas_type": "float32"}, \{"field_name": "ESC_ESCOFF", "metadata": null, "name": "ESC_ESCOFF", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ESCWorkStatus", "metadata": null, "name": "ESC_ESCWorkStatus", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ESCAlarmSig", "metadata": null, "name": "ESC_ESCAlarmSig", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_TCSCFActive", "metadata": null, "name": "ESC_TCSCFActive", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ReqIncreaseTorqueFlag", "metadata": null, "name": "ESC_ReqIncreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ReqDecreaseTorqueFlag", "metadata": null, "name": "ESC_ReqDecreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ReqIncreaseTorque", "metadata": null, "name": "ESC_ReqIncreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ReqDecreaseTorque", "metadata": null, "name": "ESC_ReqDecreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ESCValidity", "metadata": null, "name": "ESC_ESCValidity", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_RollingCount_ESC3", "metadata": null, "name": "ESC_RollingCount_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_CICkSum_ESC3", "metadata": null, "name": "ESC_CICkSum_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}], "creator": \{"library": "fastparquet", "version": "0.4.0"}, "index_columns": ["timestamps"], "pandas_version": "1.0.1"}
file schema: schema
--------------------------------------------------------------------------------
timestamps: OPTIONAL DOUBLE R:0 D:1
ESC_BrakePressure: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_BrakePressureValid: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_EBDWork: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ABSWorkLable: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_EBDAlarm: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_VehSpdValidFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ABSAlarmSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_VehSpd: OPTIONAL FLOAT R:0 D:1
ESC_FrontLeftWHeelSpd: OPTIONAL FLOAT R:0 D:1
ESC_FLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_FrontRightWHeelSpd: OPTIONAL FLOAT R:0 D:1
ESC_FRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RearLeftWheelSpd: OPTIONAL FLOAT R:0 D:1
ESC_RLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RearRightWheelSpd: OPTIONAL FLOAT R:0 D:1
ESC_RRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_Longitudinal_Acceleration_flag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_Longitudinal_Acceleration: OPTIONAL FLOAT R:0 D:1
ESC_ESCOFF: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCWorkStatus: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCAlarmSig: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_TCSCFActive: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqIncreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqDecreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqIncreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqDecreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCValidity: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RollingCount_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_CICkSum_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
row group 1: RC:67096 TS:8321051 OFFSET:4
--------------------------------------------------------------------------------
timestamps: DOUBLE GZIP DO:0 FPO:4 SZ:168467/536807/3.19 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_BrakePressure: INT32 GZIP DO:0 FPO:168549 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_BrakePressureValid: INT32 GZIP DO:0 FPO:168954 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_EBDWork: INT32 GZIP DO:0 FPO:169364 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ABSWorkLable: INT32 GZIP DO:0 FPO:169763 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_EBDAlarm: INT32 GZIP DO:0 FPO:170167 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_VehSpdValidFlag: INT32 GZIP DO:0 FPO:170567 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ABSAlarmSignal: INT32 GZIP DO:0 FPO:170974 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_VehSpd: FLOAT GZIP DO:0 FPO:171380 SZ:118126/268423/2.27 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FrontLeftWHeelSpd: FLOAT GZIP DO:0 FPO:289578 SZ:111048/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:400709 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FrontRightWHeelSpd: FLOAT GZIP DO:0 FPO:401122 SZ:110788/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:511994 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RearLeftWheelSpd: FLOAT GZIP DO:0 FPO:512407 SZ:114567/268423/2.34 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:627056 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RearRightWheelSpd: FLOAT GZIP DO:0 FPO:627469 SZ:115113/268423/2.33 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:742665 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_Longitudinal_Acceleration_flag: INT32 GZIP DO:0 FPO:743078 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_Longitudinal_Acceleration: FLOAT GZIP DO:0 FPO:743500 SZ:46863/268423/5.73 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCOFF: INT32 GZIP DO:0 FPO:790454 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCWorkStatus: INT32 GZIP DO:0 FPO:790852 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCAlarmSig: INT32 GZIP DO:0 FPO:791257 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_TCSCFActive: INT32 GZIP DO:0 FPO:791660 SZ:332/268422/808.50 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqIncreaseTorqueFlag: INT32 GZIP DO:0 FPO:792068 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqDecreaseTorqueFlag: INT32 GZIP DO:0 FPO:792481 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqIncreaseTorque: INT32 GZIP DO:0 FPO:792894 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqDecreaseTorque: INT32 GZIP DO:0 FPO:793303 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCValidity: INT32 GZIP DO:0 FPO:793712 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RollingCount_ESC3: INT32 GZIP DO:0 FPO:794115 SZ:4339/268422/61.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_CICkSum_ESC3: INT32 GZIP DO:0 FPO:798536 SZ:4341/268422/61.83 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
hive create table sql:
create external table if not exists test(`timestamps` double, `ESC_BrakePressure` int, `ESC_EBDWork` int)
stored as parquet
location '/dfmc/testcar'
TBLPROPERTIES ('PARQUET.COMPRESS'='GZIP');
load data inpath '/dfmc/testcar' into table test;
exe sql : select * from test;
error:
Failed with exception java.io.IOException:java.lang.IllegalStateException: Group type [message schema {
optional double timestamps;
optional int32 ESC_BrakePressure (INTEGER(8,false));
optional int32 ESC_EBDWork (INTEGER(8,false));
}
] does not contain requested field: optional double timestamps
> use fastparquet generate parquet format file, import hive, query error
> ------------------------------------------------------------------------
>
> Key: HIVE-23844
> URL: https://issues.apache.org/jira/browse/HIVE-23844
> Project: Hive
> Issue Type: Bug
> Components: Hive
> Affects Versions: 2.1.1
> Environment: OS: CentOS Linux release 7.6.1810
> JDK: 1.8.0_181
> hive: hive-common-2.1.1-cdh6.2.0
> fastparquet: 0.4.0
> Reporter: Henry Lu
> Priority: Major
> Attachments: 1912076_20200330_000334(102_4).parquet
>
>
> i use fastparquet generate parquet format file(Please check the attachment ！), message as follows:
> message schema
> {
> optional double timestamps;
> optional int32 ESC_BrakePressure (UINT_8);
> optional int32 ESC_BrakePressureValid (UINT_8);
> optional int32 ESC_EBDWork (UINT_8);
> optional int32 ESC_ABSWorkLable (UINT_8);
> optional int32 ESC_EBDAlarm (UINT_8);
> optional int32 ESC_VehSpdValidFlag (UINT_8);
> optional int32 ESC_ABSAlarmSignal (UINT_8);
> optional float ESC_VehSpd;
> optional float ESC_FrontLeftWHeelSpd;
> optional int32 ESC_FLWHeelSpdFaultSignal (UINT_8);
> optional float ESC_FrontRightWHeelSpd;
> optional int32 ESC_FRWHeelSpdFaultSignal (UINT_8);
> optional float ESC_RearLeftWheelSpd;
> optional int32 ESC_RLWHeelSpdFaultSignal (UINT_8);
> optional float ESC_RearRightWheelSpd;
> optional int32 ESC_RRWHeelSpdFaultSignal (UINT_8);
> optional int32 ESC_Longitudinal_Acceleration_flag (UINT_8);
> optional float ESC_Longitudinal_Acceleration;
> optional int32 ESC_ESCOFF (UINT_8);
> optional int32 ESC_ESCWorkStatus (UINT_8);
> optional int32 ESC_ESCAlarmSig (UINT_8);
> optional int32 ESC_TCSCFActive (UINT_8);
> optional int32 ESC_ReqIncreaseTorqueFlag (UINT_8);
> optional int32 ESC_ReqDecreaseTorqueFlag (UINT_8);
> optional int32 ESC_ReqIncreaseTorque (UINT_8);
> optional int32 ESC_ReqDecreaseTorque (UINT_8);
> optional int32 ESC_ESCValidity (UINT_8);
> optional int32 ESC_RollingCount_ESC3 (UINT_8);
> optional int32 ESC_CICkSum_ESC3 (UINT_8);
> }
> creator: fastparquet-python version 1.0.0 (build 111)
> extra: pandas = {"column_indexes": [
> {"field_name": null, "metadata": null, "name": null, "numpy_type": "object", "pandas_type": "mixed-integer"}
> ], "columns": [\\{"field_name": "timestamps", "metadata": null, "name": "timestamps", "numpy_type": "float64", "pandas_type": "float64"}, \\{"field_name": "ESC_BrakePressure", "metadata": null, "name": "ESC_BrakePressure", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_BrakePressureValid", "metadata": null, "name": "ESC_BrakePressureValid", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_EBDWork", "metadata": null, "name": "ESC_EBDWork", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ABSWorkLable", "metadata": null, "name": "ESC_ABSWorkLable", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_EBDAlarm", "metadata": null, "name": "ESC_EBDAlarm", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_VehSpdValidFlag", "metadata": null, "name": "ESC_VehSpdValidFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ABSAlarmSignal", "metadata": null, "name": "ESC_ABSAlarmSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_VehSpd", "metadata": null, "name": "ESC_VehSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_FrontLeftWHeelSpd", "metadata": null, "name": "ESC_FrontLeftWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_FLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_FrontRightWHeelSpd", "metadata": null, "name": "ESC_FrontRightWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_FRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_FRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_RearLeftWheelSpd", "metadata": null, "name": "ESC_RearLeftWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_RLWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_RearRightWheelSpd", "metadata": null, "name": "ESC_RearRightWheelSpd", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_RRWHeelSpdFaultSignal", "metadata": null, "name": "ESC_RRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_Longitudinal_Acceleration_flag", "metadata": null, "name": "ESC_Longitudinal_Acceleration_flag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_Longitudinal_Acceleration", "metadata": null, "name": "ESC_Longitudinal_Acceleration", "numpy_type": "float32", "pandas_type": "float32"}, \\{"field_name": "ESC_ESCOFF", "metadata": null, "name": "ESC_ESCOFF", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ESCWorkStatus", "metadata": null, "name": "ESC_ESCWorkStatus", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ESCAlarmSig", "metadata": null, "name": "ESC_ESCAlarmSig", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_TCSCFActive", "metadata": null, "name": "ESC_TCSCFActive", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqIncreaseTorqueFlag", "metadata": null, "name": "ESC_ReqIncreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqDecreaseTorqueFlag", "metadata": null, "name": "ESC_ReqDecreaseTorqueFlag", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqIncreaseTorque", "metadata": null, "name": "ESC_ReqIncreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ReqDecreaseTorque", "metadata": null, "name": "ESC_ReqDecreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_ESCValidity", "metadata": null, "name": "ESC_ESCValidity", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_RollingCount_ESC3", "metadata": null, "name": "ESC_RollingCount_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}, \\{"field_name": "ESC_CICkSum_ESC3", "metadata": null, "name": "ESC_CICkSum_ESC3", "numpy_type": "uint8", "pandas_type": "uint8"}], "creator": \{"library": "fastparquet", "version": "0.4.0"}, "index_columns": ["timestamps"], "pandas_version": "1.0.1"}
> file schema: schema
> --------------------------------------------------------------------------------
> timestamps: OPTIONAL DOUBLE R:0 D:1
> ESC_BrakePressure: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_BrakePressureValid: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_EBDWork: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ABSWorkLable: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_EBDAlarm: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_VehSpdValidFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ABSAlarmSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_VehSpd: OPTIONAL FLOAT R:0 D:1
> ESC_FrontLeftWHeelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_FLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_FrontRightWHeelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_FRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_RearLeftWheelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_RLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_RearRightWheelSpd: OPTIONAL FLOAT R:0 D:1
> ESC_RRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_Longitudinal_Acceleration_flag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_Longitudinal_Acceleration: OPTIONAL FLOAT R:0 D:1
> ESC_ESCOFF: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ESCWorkStatus: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ESCAlarmSig: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_TCSCFActive: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqIncreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqDecreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqIncreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ReqDecreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_ESCValidity: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_RollingCount_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
> ESC_CICkSum_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
> row group 1: RC:67096 TS:8321051 OFFSET:4
> --------------------------------------------------------------------------------
> timestamps: DOUBLE GZIP DO:0 FPO:4 SZ:168467/536807/3.19 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_BrakePressure: INT32 GZIP DO:0 FPO:168549 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_BrakePressureValid: INT32 GZIP DO:0 FPO:168954 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_EBDWork: INT32 GZIP DO:0 FPO:169364 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ABSWorkLable: INT32 GZIP DO:0 FPO:169763 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_EBDAlarm: INT32 GZIP DO:0 FPO:170167 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_VehSpdValidFlag: INT32 GZIP DO:0 FPO:170567 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ABSAlarmSignal: INT32 GZIP DO:0 FPO:170974 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_VehSpd: FLOAT GZIP DO:0 FPO:171380 SZ:118126/268423/2.27 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FrontLeftWHeelSpd: FLOAT GZIP DO:0 FPO:289578 SZ:111048/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:400709 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FrontRightWHeelSpd: FLOAT GZIP DO:0 FPO:401122 SZ:110788/268423/2.42 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_FRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:511994 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RearLeftWheelSpd: FLOAT GZIP DO:0 FPO:512407 SZ:114567/268423/2.34 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:627056 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RearRightWheelSpd: FLOAT GZIP DO:0 FPO:627469 SZ:115113/268423/2.33 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:742665 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_Longitudinal_Acceleration_flag: INT32 GZIP DO:0 FPO:743078 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_Longitudinal_Acceleration: FLOAT GZIP DO:0 FPO:743500 SZ:46863/268423/5.73 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCOFF: INT32 GZIP DO:0 FPO:790454 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCWorkStatus: INT32 GZIP DO:0 FPO:790852 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCAlarmSig: INT32 GZIP DO:0 FPO:791257 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_TCSCFActive: INT32 GZIP DO:0 FPO:791660 SZ:332/268422/808.50 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqIncreaseTorqueFlag: INT32 GZIP DO:0 FPO:792068 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqDecreaseTorqueFlag: INT32 GZIP DO:0 FPO:792481 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqIncreaseTorque: INT32 GZIP DO:0 FPO:792894 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ReqDecreaseTorque: INT32 GZIP DO:0 FPO:793303 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_ESCValidity: INT32 GZIP DO:0 FPO:793712 SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_RollingCount_ESC3: INT32 GZIP DO:0 FPO:794115 SZ:4339/268422/61.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
> ESC_CICkSum_ESC3: INT32 GZIP DO:0 FPO:798536 SZ:4341/268422/61.83 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
>
> hive create table sql:
> create external table if not exists test(`timestamps` double, `ESC_BrakePressure` int, `ESC_EBDWork` int)
> stored as parquet
> location '/dfmc/testcar'
> TBLPROPERTIES ('PARQUET.COMPRESS'='GZIP');
>
> load data inpath '/dfmc/testcar' into table test;
>
> exe sql : select * from test;
>
> error:
> Failed with exception java.io.IOException:java.lang.IllegalStateException: Group type [message schema
> { optional double timestamps; optional int32 ESC_BrakePressure (INTEGER(8,false)); optional int32 ESC_EBDWork (INTEGER(8,false)); }
> ] does not contain requested field: optional double timestamps
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)