You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2021/01/06 06:28:07 UTC
[GitHub] [incubator-doris] StevenTangJian opened a new issue #5196: Spark Load Hive Table to Doris
StevenTangJian opened a new issue #5196:
URL: https://github.com/apache/incubator-doris/issues/5196
**first**
I have a hive outer table schema definition in drois :
--创建 外部hive表
create EXTERNAL table exit_jour_result_dt (
n_en_date int comment '入口日期',
n_en_time int comment '入口时间',
c_en_shift_code varchar(6) DEFAULT NULL comment '入口流水',
n_en_station_id int comment '入口站点编码',
n_en_card_id varchar(16) DEFAULT NULL comment '入口工号',
c_en_vehicle_class varchar(6) DEFAULT NULL comment '车型 1-一型客车,2-二型客车,3-三型客车,4-四型客车,11-一型货车,12-二型货车,13-三型货车,14-四型货车,15-五型货车,16-六型货车,21-一型专项作业车,22-二型专项作业车,23-三型专项作业车,24-四型专项作业车,25-五型专项作业车,26-六型专项作业车',
n_ex_date int comment '出口统计日期',
n_ex_time int comment '出口时间',
c_ex_shift_code varchar(6) DEFAULT NULL comment '出口班次',
n_ex_op_code int comment '出口工号',
c_ex_vehicle_class varchar(6) DEFAULT NULL comment '出口车型 同入口车型',
c_ex_vehicle_type varchar(6) DEFAULT NULL comment '出口车种 ‘1’—普通车，‘2’—公务车（含我系统免费卡车），‘3’—冲卡车，‘4’—紧急车（即我系统车队），‘5’—拖车，‘6’—U型车（收费的和不收费的），‘7’—超时车，‘8’—违章车，‘9’—纸券车，‘A’—免费车（即我系统特殊车），‘B’—免费卡车（结算中心免费卡），‘C’—月票卡车，‘D’—统缴卡车，‘E’—绿色通道车（），‘F’—政策减免车',
c_ex_card_status varchar(6) DEFAULT NULL comment '卡状态 ‘0’-正常有卡，‘1’-不可读卡，‘2’-坏卡，‘3’-无卡',
c_other_type varchar(22) DEFAULT NULL comment '其他卡类型（状态），含义见后',
n_other_id varchar(22) DEFAULT NULL comment '其他卡号，公务卡、预付卡等',
vc_marks varchar(170) DEFAULT NULL comment '路径标志',
d_toll double DEFAULT NULL comment '交易金额 d_toll+d_agency_toll 实际出口收费金额',
d_unpaid double DEFAULT NULL comment '未付金额（免费金额）',
d_forfeit double DEFAULT NULL comment '罚款金额',
d_agency_toll double DEFAULT NULL comment '代收费金额',
vc_agencys varchar(10) DEFAULT NULL comment '代收费代码，可叠加',
c_category varchar(6) DEFAULT NULL comment '客货车，’0’-客，’1’-货 2-专车 ',
c_pay_way varchar(6) DEFAULT NULL comment '付费方式，’0’不付款,’1’.现金、’2’.公务卡（联网公务卡、沪杭甬公务卡、免费卡、纪念卡），’3’.储值卡，’4’.银联卡，’5’-免费卡，’6’统缴卡,’7’路段（沪杭甬）预付卡，’8’路段信用卡？’9’绿色通道，’A’政策减免，’B’记账卡, D-路段闪付卡, E-路段支付宝',
n_bill_no BIGINT comment '发票号码',
d_svc_balance double comment '-储值卡余额',
n_prn_times int comment '重打次数',
n_box_id int comment '出口实际日期',
n_zone_time int comment '小时时间段',
n_card_lane_id int comment '进口车道号 前4位站点码',
n_card_serial_no int comment '出口流水',
n_card_op_code int comment '出口工号',
c_card_license varchar(20) DEFAULT NULL comment '卡内车牌号',
c_ex_license varchar(20) DEFAULT NULL comment '出口车牌',
d_card_toll double comment '按卡中车型的通行费',
d_card_agency_toll double comment '按卡中车型的代收费',
c_part_vehicle_type varchar(8) DEFAULT NULL comment '过程车种，可组合，祥见后',
c_special_type varchar(8) DEFAULT NULL comment '特殊处理类型，祥见后',
vc_card_agencys varchar(8) DEFAULT NULL comment '按卡中车型的代收费代码 轴型',
d_card_forfeit double comment '按卡中车型罚款，一般=0？',
n_temp int comment '未知，一般=5？',
c_temp varchar(10) DEFAULT NULL comment '0省外绿通，1-电力抢修，2省内绿通，3货车，’X’-etc出，’Y’-',
c_cardno varchar(19) DEFAULT NULL comment '银联卡号',
c_bankexamno varchar(19) DEFAULT NULL comment '银联卡号',
c_posserialno varchar(12) DEFAULT NULL comment 'pos机流水号',
d_weight double comment '车重',
d_over_weight double comment '超重',
c_ticket_type varchar(6) DEFAULT NULL comment '票种0是无卡\2纸券\3免费票\4复合通行卡；5ETC卡',
c_verifycode varchar(8) DEFAULT NULL comment '校验码',
n_ex_psam_id int comment '出口sam卡标识，=1',
n_en_psam_id int comment '入口sam卡标识，=0，1',
c_lane_type varchar(6) DEFAULT NULL comment '出口车道类型，说明见后 2 MTC 4 ETC ',
c_cacu_way varchar(6) DEFAULT NULL comment '计费方式',
vc_tac varchar(8) comment 'TAC码',
vc_issue_code varchar(8) DEFAULT NULL comment '发行代码',
d_fee_length double comment '计费总里程数',
d_fare0 double comment '里程费，不含代收费',
d_fare1 double comment '叠加费',
d_fare2 double comment '轴限',
vc_fix_marks varchar(100) comment '修正标识',
vc_fix_marks_status varchar(50) comment '标识修正状态',
n_ex_cpu_psamid int comment 'PSAM 卡脱机交易序列号',
n_vehicle_seats int comment '座位数',
n_trade_time int comment 'n_trade_time',
n_trade_speed int comment 'n_trade_speed',
n_batch_no int comment '批次',
n_temp1 int comment '保留',
c_temp1 varchar(10) DEFAULT NULL comment '保留',
n_temp2 int comment '保留',
c_temp2 varchar(10) DEFAULT NULL comment '保留',
c_make_ticket varchar(6) DEFAULT NULL comment '开盘标识',
n_en_sys_date int comment '入口进站系统日期',
n_favouredpolicy_type int comment '优惠政策类型 政策优惠0-不优惠；1-宁波绕城优惠；2-G20优惠；3-ETC客车优惠；4-舟山优惠；',
n_mb_trade_date int comment '支付日期 移动端交易日期',
n_mb_trade_time int comment '支付时间',
vc_temp3 varchar(10) DEFAULT NULL comment '预留',
vc_iden_license varchar(20) comment '识别车牌号码',
n_marks_mode int comment '路标计费模式',
d_bef_balance double comment '交易前余额',
c_en_lane_type varchar(6) DEFAULT NULL comment '入口车道类型 1 MTC 3 ETC',
c_en_category varchar(6) DEFAULT NULL comment '入口客货类型',
n_station int comment '收费站出站站点码',
n_ex_serial_no int comment '出口流水号',
n_ex_lane_id int comment '出口车道号',
c_send_flag2 int comment '收费站传输标志，0-未确定1未发送，2-已发送',
c_send_flag3 varchar(10) comment '分中心发送标志，0不确定，1未发，2已发',
c_send_flag1 varchar(10) comment '结算中心传输标志，0-未确定1未发送，2-已发送',
vc_marks1 varchar(100) comment 'vc_marks1',
n_axis int comment '轴数',
c_send_flag4 varchar(10) comment 'c_send_flag4',
vc_axle_specialtype varchar(20) comment '车轴特殊类型',
n_exaxlecount int comment '出口轴数',
vc_cardver varchar(2) comment '卡片版本',
n_algorithmidentifier int comment '算法描述符',
vc_stationhex varchar(8) comment '收费站 同出站站点编码(16进制)',
vc_exhex varchar(10) DEFAULT NULL comment '出站站点编码(16进制)',
vc_enhex varchar(10) DEFAULT NULL comment '进站站点编码(16进制)',
d_enweight double comment '入口重量',
n_enaxlecount int comment '入口轴数',
vc_enaxletype varchar(6) DEFAULT NULL comment '入口轴类型',
d_enpermittedweight double DEFAULT NULL comment '入口限重',
n_enspecialtruck int comment '特种卡车',
vc_markhex varchar(512) comment '路径Hex串',
d_originaltotalfee double comment '原始总金额',
d_originalpassfare double comment '原始通行费',
d_originalagencyfare double comment '',
d_originalfare0 double comment '原价0',
d_originalfare1 double comment '原价1',
d_originalfare2 double comment '原价2',
vc_rateversion varchar(40) comment '计费模块版本号',
vc_obumac varchar(8) comment 'OBU/CPC MAC地址',
vc_obusn varchar(20) comment 'obusn',
n_electricalpercent int comment 'CPC卡电量百分比',
n_multiprovince int comment '是否多省交易 0 没有 1 有',
inputtime varchar(40) comment '写入时间',
vc_pass_id varchar(40) comment '方向编号(2位)+cpc或etc卡的编号(20位)+通行时间(年月日时分秒)',
vc_obu_id varchar(20) comment 'OBU ID',
vc_ex_lane_hex varchar(10) comment '出站车道号hex值',
vc_en_lane_hex varchar(10) comment '进站车道号hex值',
n_toll_support int comment '计费支撑',
n_obu_sign int comment 'OBU 单/双片 标识 1-单片式 OBU 2-双片式 OBU',
n_en_vehicle_usertype int comment '入口车辆用户类型',
n_ex_vehicle_usertype int comment '出口车辆用户类型',
n_direction int comment '',
vc_vehicle_sign_id varchar(200) comment '关联图片流水号',
d_trans_fee float comment '卡片代收金额',
n_sign_status int comment '',
vc_trade_id char(38) NOT NULL comment '前21位 出口车道+时间 出口处理编号',
ts_timestamp bigint comment '出口处理时间',
n_vehicle_sign int comment '',
n_vehicle_class_gb int comment '',
vc_cert_no varchar(50) DEFAULT NULL comment '',
vc_special_type_gb varchar(50) comment '特情类型（部标）',
n_gantry_lines int comment '',
n_detail_lines int comment '',
d_zj_fee float comment '',
n_obu_count_local int comment '总交易成功次数(门架)',
n_obu_count_total int comment '',
n_obu_count_nocard int comment '标签无卡累计次数',
n_obu_count_provs int comment '通行省份个数',
vc_obu_en_station varchar(10) DEFAULT NULL comment 'OBU省内入口站码',
n_obu_distance bigint comment 'obu内的里程',
d_prov_pay_fee float comment '省内应收金额',
d_prov_fee float comment '省内实收金额',
d_cpu_fee_total float comment '',
vc_obu_ef04_rec varchar(200) comment '',
vc_prov_ids varchar(100) comment '行程跨省届标识',
vc_prov_fees varchar(200) comment '',
vc_prov_od varchar(1500) comment 'CPC分省OD明细',
vc_short_fare_ver varchar(40) DEFAULT NULL comment '',
d_short_fee_total float DEFAULT NULL comment '最小费额交易金额',
n_agency_gantry_num int comment '',
d_agency_gantry_fee float comment '',
n_display_amount_type int comment '',
n_trans_pay_type int comment '',
d_short_fee_mileage float comment '最小费额里程',
d_fee_rate float comment '交易金额占比/100',
n_exit_fee_type int comment '计费方式 -1 按OBU内 累计优惠后金额计费 -2 按OBU内累计优惠前金额计费 -3 按CPC卡内累计金额计费 -4 省中心在线服务计费 -5 部中心在线服务计费 -6 按全网最小费额计费 -11 开放式计费',
d_total_pay_fee float comment '应收金额(元)',
d_total_fee float comment '总交易金额(元)',
n_trade_mode int comment '交易类型 0、1、未知 2、ETC小天线 3、手持天线（手持机',
vc_appoint varchar(50) comment '预约ID',
d_prov_pass_fee float DEFAULT NULL comment '省内通行费实收',
ds int comment '日期分区'
)
ENGINE=hive
properties
(
"database" = "hhy_dw",
"table" = "ods_chg_rt_exit_jour_result_dt",
"hive.metastore.uris" = "thrift://33.69.6.24:9083"
);
**then i definition doris table as below:**
Create TABLE `ods_exit_jour` (
n_en_date int comment '入口日期',
n_en_time int comment '入口时间',
c_en_shift_code varchar(6) DEFAULT NULL comment '入口流水',
n_en_station_id int comment '入口站点编码',
n_en_card_id varchar(16) DEFAULT NULL comment '入口工号',
c_en_vehicle_class varchar(6) DEFAULT NULL comment '车型 1-一型客车,2-二型客车,3-三型客车,4-四型客车,11-一型货车,12-二型货车,13-三型货车,14-四型货车,15-五型货车,16-六型货车,21-一型专项作业车,22-二型专项作业车,23-三型专项作业车,24-四型专项作业车,25-五型专项作业车,26-六型专项作业车',
n_ex_date int comment '出口统计日期',
n_ex_time int comment '出口时间',
c_ex_shift_code varchar(6) DEFAULT NULL comment '出口班次',
n_ex_op_code int comment '出口工号',
c_ex_vehicle_class varchar(6) DEFAULT NULL comment '出口车型 同入口车型',
c_ex_vehicle_type varchar(6) DEFAULT NULL comment '出口车种 ‘1’—普通车，‘2’—公务车（含我系统免费卡车），‘3’—冲卡车，‘4’—紧急车（即我系统车队），‘5’—拖车，‘6’—U型车（收费的和不收费的），‘7’—超时车，‘8’—违章车，‘9’—纸券车，‘A’—免费车（即我系统特殊车），‘B’—免费卡车（结算中心免费卡），‘C’—月票卡车，‘D’—统缴卡车，‘E’—绿色通道车（），‘F’—政策减免车',
c_ex_card_status varchar(6) DEFAULT NULL comment '卡状态 ‘0’-正常有卡，‘1’-不可读卡，‘2’-坏卡，‘3’-无卡',
c_other_type varchar(22) DEFAULT NULL comment '其他卡类型（状态），含义见后',
n_other_id varchar(22) DEFAULT NULL comment '其他卡号，公务卡、预付卡等',
vc_marks varchar(170) DEFAULT NULL comment '路径标志',
d_toll double DEFAULT NULL comment '交易金额 d_toll+d_agency_toll 实际出口收费金额',
d_unpaid double DEFAULT NULL comment '未付金额（免费金额）',
d_forfeit double DEFAULT NULL comment '罚款金额',
d_agency_toll double DEFAULT NULL comment '代收费金额',
vc_agencys varchar(10) DEFAULT NULL comment '代收费代码，可叠加',
c_category varchar(6) DEFAULT NULL comment '客货车，’0’-客，’1’-货 2-专车 ',
c_pay_way varchar(6) DEFAULT NULL comment '付费方式，’0’不付款,’1’.现金、’2’.公务卡（联网公务卡、沪杭甬公务卡、免费卡、纪念卡），’3’.储值卡，’4’.银联卡，’5’-免费卡，’6’统缴卡,’7’路段（沪杭甬）预付卡，’8’路段信用卡？’9’绿色通道，’A’政策减免，’B’记账卡, D-路段闪付卡, E-路段支付宝',
n_bill_no BIGINT comment '发票号码',
d_svc_balance double comment '-储值卡余额',
n_prn_times int comment '重打次数',
n_box_id int comment '出口实际日期',
n_zone_time int comment '小时时间段',
n_card_lane_id int comment '进口车道号 前4位站点码',
n_card_serial_no int comment '出口流水',
n_card_op_code int comment '出口工号',
c_card_license varchar(20) DEFAULT NULL comment '卡内车牌号',
c_ex_license varchar(20) DEFAULT NULL comment '出口车牌',
d_card_toll double comment '按卡中车型的通行费',
d_card_agency_toll double comment '按卡中车型的代收费',
c_part_vehicle_type varchar(8) DEFAULT NULL comment '过程车种，可组合，祥见后',
c_special_type varchar(8) DEFAULT NULL comment '特殊处理类型，祥见后',
vc_card_agencys varchar(8) DEFAULT NULL comment '按卡中车型的代收费代码 轴型',
d_card_forfeit double comment '按卡中车型罚款，一般=0？',
n_temp int comment '未知，一般=5？',
c_temp varchar(10) DEFAULT NULL comment '0省外绿通，1-电力抢修，2省内绿通，3货车，’X’-etc出，’Y’-',
c_cardno varchar(19) DEFAULT NULL comment '银联卡号',
c_bankexamno varchar(19) DEFAULT NULL comment '银联卡号',
c_posserialno varchar(12) DEFAULT NULL comment 'pos机流水号',
d_weight double comment '车重',
d_over_weight double comment '超重',
c_ticket_type varchar(6) DEFAULT NULL comment '票种0是无卡\2纸券\3免费票\4复合通行卡；5ETC卡',
c_verifycode varchar(8) DEFAULT NULL comment '校验码',
n_ex_psam_id int comment '出口sam卡标识，=1',
n_en_psam_id int comment '入口sam卡标识，=0，1',
c_lane_type varchar(6) DEFAULT NULL comment '出口车道类型，说明见后 2 MTC 4 ETC ',
c_cacu_way varchar(6) DEFAULT NULL comment '计费方式',
vc_tac varchar(8) comment 'TAC码',
vc_issue_code varchar(8) DEFAULT NULL comment '发行代码',
d_fee_length double comment '计费总里程数',
d_fare0 double comment '里程费，不含代收费',
d_fare1 double comment '叠加费',
d_fare2 double comment '轴限',
vc_fix_marks varchar(100) comment '修正标识',
vc_fix_marks_status varchar(50) comment '标识修正状态',
n_ex_cpu_psamid int comment 'PSAM 卡脱机交易序列号',
n_vehicle_seats int comment '座位数',
n_trade_time int comment 'n_trade_time',
n_trade_speed int comment 'n_trade_speed',
n_batch_no int comment '批次',
n_temp1 int comment '保留',
c_temp1 varchar(10) DEFAULT NULL comment '保留',
n_temp2 int comment '保留',
c_temp2 varchar(10) DEFAULT NULL comment '保留',
c_make_ticket varchar(6) DEFAULT NULL comment '开盘标识',
n_en_sys_date int comment '入口进站系统日期',
n_favouredpolicy_type int comment '优惠政策类型 政策优惠0-不优惠；1-宁波绕城优惠；2-G20优惠；3-ETC客车优惠；4-舟山优惠；',
n_mb_trade_date int comment '支付日期 移动端交易日期',
n_mb_trade_time int comment '支付时间',
vc_temp3 varchar(10) DEFAULT NULL comment '预留',
vc_iden_license varchar(20) comment '识别车牌号码',
n_marks_mode int comment '路标计费模式',
d_bef_balance double comment '交易前余额',
c_en_lane_type varchar(6) DEFAULT NULL comment '入口车道类型 1 MTC 3 ETC',
c_en_category varchar(6) DEFAULT NULL comment '入口客货类型',
n_station int comment '收费站出站站点码',
n_ex_serial_no int comment '出口流水号',
n_ex_lane_id int comment '出口车道号',
c_send_flag2 int comment '收费站传输标志，0-未确定1未发送，2-已发送',
c_send_flag3 varchar(10) comment '分中心发送标志，0不确定，1未发，2已发',
c_send_flag1 varchar(10) comment '结算中心传输标志，0-未确定1未发送，2-已发送',
vc_marks1 varchar(100) comment 'vc_marks1',
n_axis int comment '轴数',
c_send_flag4 varchar(10) comment 'c_send_flag4',
vc_axle_specialtype varchar(20) comment '车轴特殊类型',
n_exaxlecount int comment '出口轴数',
vc_cardver varchar(2) comment '卡片版本',
n_algorithmidentifier int comment '算法描述符',
vc_stationhex varchar(8) comment '收费站 同出站站点编码(16进制)',
vc_exhex varchar(10) DEFAULT NULL comment '出站站点编码(16进制)',
vc_enhex varchar(10) DEFAULT NULL comment '进站站点编码(16进制)',
d_enweight double comment '入口重量',
n_enaxlecount int comment '入口轴数',
vc_enaxletype varchar(6) DEFAULT NULL comment '入口轴类型',
d_enpermittedweight double DEFAULT NULL comment '入口限重',
n_enspecialtruck int comment '特种卡车',
vc_markhex varchar(512) comment '路径Hex串',
d_originaltotalfee double comment '原始总金额',
d_originalpassfare double comment '原始通行费',
d_originalagencyfare double comment '',
d_originalfare0 double comment '原价0',
d_originalfare1 double comment '原价1',
d_originalfare2 double comment '原价2',
vc_rateversion varchar(40) comment '计费模块版本号',
vc_obumac varchar(8) comment 'OBU/CPC MAC地址',
vc_obusn varchar(20) comment 'obusn',
n_electricalpercent int comment 'CPC卡电量百分比',
n_multiprovince int comment '是否多省交易 0 没有 1 有',
inputtime varchar(40) comment '写入时间',
vc_pass_id varchar(40) comment '方向编号(2位)+cpc或etc卡的编号(20位)+通行时间(年月日时分秒)',
vc_obu_id varchar(20) comment 'OBU ID',
vc_ex_lane_hex varchar(10) comment '出站车道号hex值',
vc_en_lane_hex varchar(10) comment '进站车道号hex值',
n_toll_support int comment '计费支撑',
n_obu_sign int comment 'OBU 单/双片 标识 1-单片式 OBU 2-双片式 OBU',
n_en_vehicle_usertype int comment '入口车辆用户类型',
n_ex_vehicle_usertype int comment '出口车辆用户类型',
n_direction int comment '',
vc_vehicle_sign_id varchar(200) comment '关联图片流水号',
d_trans_fee float comment '卡片代收金额',
n_sign_status int comment '',
vc_trade_id char(38) NOT NULL comment '前21位 出口车道+时间 出口处理编号',
ts_timestamp bigint comment '出口处理时间',
n_vehicle_sign int comment '',
n_vehicle_class_gb int comment '',
vc_cert_no varchar(50) DEFAULT NULL comment '',
vc_special_type_gb varchar(50) comment '特情类型（部标）',
n_gantry_lines int comment '',
n_detail_lines int comment '',
d_zj_fee float comment '',
n_obu_count_local int comment '总交易成功次数(门架)',
n_obu_count_total int comment '',
n_obu_count_nocard int comment '标签无卡累计次数',
n_obu_count_provs int comment '通行省份个数',
vc_obu_en_station varchar(10) DEFAULT NULL comment 'OBU省内入口站码',
n_obu_distance bigint comment 'obu内的里程',
d_prov_pay_fee float **comment** '省内应收金额',
d_prov_fee float comment '省内实收金额',
d_cpu_fee_total float comment '',
vc_obu_ef04_rec varchar(200) comment '',
vc_prov_ids varchar(100) comment '行程跨省届标识',
vc_prov_fees varchar(200) comment '',
vc_prov_od varchar(1500) comment 'CPC分省OD明细',
vc_short_fare_ver varchar(40) DEFAULT NULL comment '',
d_short_fee_total float DEFAULT NULL comment '最小费额交易金额',
n_agency_gantry_num int comment '',
d_agency_gantry_fee float comment '',
n_display_amount_type int comment '',
n_trans_pay_type int comment '',
d_short_fee_mileage float comment '最小费额里程',
d_fee_rate float comment '交易金额占比/100',
n_exit_fee_type int comment '计费方式 -1 按OBU内 累计优惠后金额计费 -2 按OBU内累计优惠前金额计费 -3 按CPC卡内累计金额计费 -4 省中心在线服务计费 -5 部中心在线服务计费 -6 按全网最小费额计费 -11 开放式计费',
d_total_pay_fee float comment '应收金额(元)',
d_total_fee float comment '总交易金额(元)',
n_trade_mode int comment '交易类型 0、1、未知 2、ETC小天线 3、手持天线（手持机',
vc_appoint varchar(50) comment '预约ID',
d_prov_pass_fee float DEFAULT NULL comment '省内通行费实收',
ds int comment '日期分区'
) ENGINE=olap
COMMENT " test doris table"
DISTRIBUTED BY HASH(`vc_trade_id`) BUCKETS 12
PROPERTIES (
"storage_type"="column",
"replication_num" = "1"
);
**finally I want to load exit_jour_result_dt(hive table) to ods_exit_jour**
LOAD LABEL hhy_dw.test_spark_load
(
DATA FROM TABLE exit_jour_result_dt
INTO TABLE ods_exit_jour
where ds=20210105
)
WITH RESOURCE 'spark_res'
(
"spark.executor.memory" = "4g",
"spark.shuffle.compress" = "true"
)
PROPERTIES
(
"timeout" = "3600"
);
but it produce an error on spark program as blew:
21/01/06 14:11:23 ERROR AsyncEventQueue: Listener EventLoggingListener threw an exception
java.lang.ClassCastException: java.lang.String cannot be cast to java.util.List
at org.apache.spark.util.JsonProtocol$.accumValueToJson(JsonProtocol.scala:343)
at org.apache.spark.util.JsonProtocol$$anonfun$accumulableInfoToJson$3.apply(JsonProtocol.scala:319)
at org.apache.spark.util.JsonProtocol$$anonfun$accumulableInfoToJson$3.apply(JsonProtocol.scala:319)
at scala.Option.map(Option.scala:146)
at org.apache.spark.util.JsonProtocol$.accumulableInfoToJson(JsonProtocol.scala:319)
at org.apache.spark.util.JsonProtocol$$anonfun$accumulablesToJson$2.apply(JsonProtocol.scala:312)
at org.apache.spark.util.JsonProtocol$$anonfun$accumulablesToJson$2.apply(JsonProtocol.scala:312)
at scala.collection.immutable.List.map(List.scala:288)
at org.apache.spark.util.JsonProtocol$.accumulablesToJson(JsonProtocol.scala:312)
at org.apache.spark.util.JsonProtocol$.taskInfoToJson(JsonProtocol.scala:304)
at org.apache.spark.util.JsonProtocol$.taskEndToJson(JsonProtocol.scala:148)
at org.apache.spark.util.JsonProtocol$.sparkEventToJson(JsonProtocol.scala:77)
at org.apache.spark.scheduler.EventLoggingListener.org$apache$spark$scheduler$EventLoggingListener$$logEvent(EventLoggingListener.scala:143)
at org.apache.spark.scheduler.EventLoggingListener.onTaskEnd(EventLoggingListener.scala:170)
at org.apache.spark.scheduler.SparkListenerBus$class.doPostEvent(SparkListenerBus.scala:45)
at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)
at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:37)
at org.apache.spark.util.ListenerBus$class.postToAll(ListenerBus.scala:91)
how can I solve this problem
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] wangbo commented on issue #5196: Spark Load Hive Table to Doris
Posted by GitBox <gi...@apache.org>.
wangbo commented on issue #5196:
URL: https://github.com/apache/incubator-doris/issues/5196#issuecomment-755114190
This stack doesn't contain doris's stack.
So it's hard to judge.
Can you provide more stack which contains keyword 'doris'?
Or maybe a big file which contains spark driver's total log is better
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] StevenTangJian closed issue #5196: Spark Load Hive Table to Doris
Posted by GitBox <gi...@apache.org>.
StevenTangJian closed issue #5196:
URL: https://github.com/apache/incubator-doris/issues/5196
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] StevenTangJian commented on issue #5196: Spark Load Hive Table to Doris
Posted by GitBox <gi...@apache.org>.
StevenTangJian commented on issue #5196:
URL: https://github.com/apache/incubator-doris/issues/5196#issuecomment-755151321
> This stack doesn't contain doris's stack.
> So it's hard to judge.
> Can you provide more stack which contains keyword 'doris'?
> Or maybe a big file which contains spark driver's total log is better
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] StevenTangJian removed a comment on issue #5196: Spark Load Hive Table to Doris
Posted by GitBox <gi...@apache.org>.
StevenTangJian removed a comment on issue #5196:
URL: https://github.com/apache/incubator-doris/issues/5196#issuecomment-755151321
> This stack doesn't contain doris's stack.
> So it's hard to judge.
> Can you provide more stack which contains keyword 'doris'?
> Or maybe a big file which contains spark driver's total log is better
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org