You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by GitBox <gi...@apache.org> on 2022/10/28 08:13:52 UTC
[GitHub] [dolphinscheduler] mobiuschh commented on issue #12590: [Bug] [data-quality] Data quality task can't run
mobiuschh commented on issue #12590:
URL: https://github.com/apache/dolphinscheduler/issues/12590#issuecomment-1294664274
[LOG-PATH]: /opt/dolphinscheduler/logs/20221028/7359692479072_2-17893-46126.log, [HOST]: Host{address='dolphinscheduler-worker-2.dolphinscheduler-worker-headless:1234', ip='dolphinscheduler-worker-2.dolphinscheduler-worker-headless', port=1234}
[INFO] 2022-10-28 11:37:37.070 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[83] - data quality task params {"localParams":[],"resourceList":[],"ruleId":1,"ruleInputParameter":{"check_type":"1","comparison_type":1,"comparison_name":"0","failure_strategy":"0","operator":"3","src_connector_type":0,"src_datasource_id":8,"src_field":"id","src_table":"ads_search_store_hotword_i_d","threshold":"0"},"sparkParameters":{"deployMode":"cluster","driverCores":1,"driverMemory":"512M","executorCores":2,"executorMemory":"2G","numExecutors":2,"others":"--conf spark.yarn.maxAppAttempts=1"}}
[INFO] 2022-10-28 11:37:37.072 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[181] - data quality task command: ${SPARK_HOME2}/bin/spark-submit --master yarn --deploy-mode cluster --driver-cores 1 --driver-memory 512M --num-executors 2 --executor-cores 2 --executor-memory 2G --queue default --conf spark.yarn.maxAppAttempts=1 /opt/dolphinscheduler/libs/dolphinscheduler-data-quality-dev-SNAPSHOT.jar "{\"name\":\"$t(null_check)\",\"env\":{\"type\":\"batch\",\"config\":null},\"readers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"stat\",\"password\":\"***********\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"bidata\",\"output_table\":\"stat_ads_search_store_hotword_i_d\",\"table\":\"ads_search_store_hotword_i_d\",\"url\":\"jdbc:mysql://am-wz99d8065a99x5f0w167320o.ads.aliyuncs.com:3306/stat?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=f
alse\"} }],\"transformers\":[{\"type\":\"sql\",\"config\":{\"index\":1,\"output_table\":\"null_items\",\"sql\":\"SELECT * FROM stat_ads_search_store_hotword_i_d WHERE (id is null or id = '') \"} },{\"type\":\"sql\",\"config\":{\"index\":2,\"output_table\":\"null_count\",\"sql\":\"SELECT COUNT(*) AS nulls FROM null_items\"} }],\"writers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_execute_result\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select 0 as rule_type,'$t(null_check)' as rule_name,0 as process_definition_id,17893 as process_instance_id,46126 as task_instance_id,null_count.nulls AS statistics_value,0 AS comparison_value,1 AS comparison_type,1 as check_ty
pe,0 as threshold,3 as operator,0 as failure_strategy,'hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B\u8BD5' as error_output_path,'2022-10-28 11:37:37' as create_time,'2022-10-28 11:37:37' as update_time from null_count \"} },{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_task_statistics_value\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select 0 as process_definition_id,46126 as task_instance_id,1 as rule_id,'3EHCYI2MVDDNF1CFUGCCEWXVTHXW+UB0PYME6EZPC3Y=' as unique_code,'null_count.nulls'AS statistics_name,null_count.nulls AS statistics_value,'2022-10-28 11:37:37' as data_time,'2022-10-28 11:37:37' as create_time,
'2022-10-28 11:37:37' as update_time from null_count\"} },{\"type\":\"hdfs_file\",\"config\":{\"path\":\"hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B\u8BD5\",\"input_table\":\"null_items\"} }]}"
[INFO] 2022-10-28 11:37:37.073 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[85] - tenantCode user:dolphin, task dir:17893_46126
[INFO] 2022-10-28 11:37:37.073 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[90] - create command file:/tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126/17893_46126.command
[INFO] 2022-10-28 11:37:37.073 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[116] - command : #!/bin/sh
BASEDIR=$(cd `dirname $0`; pwd)
cd $BASEDIR
source /opt/dolphinscheduler/conf/dolphinscheduler_env.sh
${SPARK_HOME2}/bin/spark-submit --master yarn --deploy-mode cluster --driver-cores 1 --driver-memory 512M --num-executors 2 --executor-cores 2 --executor-memory 2G --queue default --conf spark.yarn.maxAppAttempts=1 /opt/dolphinscheduler/libs/dolphinscheduler-data-quality-dev-SNAPSHOT.jar "{\"name\":\"$t(null_check)\",\"env\":{\"type\":\"batch\",\"config\":null},\"readers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"stat\",\"password\":\"***********\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"bidata\",\"output_table\":\"stat_ads_search_store_hotword_i_d\",\"table\":\"ads_search_store_hotword_i_d\",\"url\":\"jdbc:mysql://am-wz99d8065a99x5f0w167320o.ads.aliyuncs.com:3306/stat?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\"} }],\"transformers\":[{\"type\":\"sql\",\"config\":{\"index\":1,\"output_table\":\"null_items\",\"sql\":\"SELECT * FROM stat_ads_search_store_hotword_i_d WHERE (id is null or id = '') \"} },{\"type\
":\"sql\",\"config\":{\"index\":2,\"output_table\":\"null_count\",\"sql\":\"SELECT COUNT(*) AS nulls FROM null_items\"} }],\"writers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_execute_result\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select 0 as rule_type,'$t(null_check)' as rule_name,0 as process_definition_id,17893 as process_instance_id,46126 as task_instance_id,null_count.nulls AS statistics_value,0 AS comparison_value,1 AS comparison_type,1 as check_type,0 as threshold,3 as operator,0 as failure_strategy,'hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B\u8BD5' as error_output_path,'2022-10-28 11:37:37'
as create_time,'2022-10-28 11:37:37' as update_time from null_count \"} },{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_task_statistics_value\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select 0 as process_definition_id,46126 as task_instance_id,1 as rule_id,'3EHCYI2MVDDNF1CFUGCCEWXVTHXW+UB0PYME6EZPC3Y=' as unique_code,'null_count.nulls'AS statistics_name,null_count.nulls AS statistics_value,'2022-10-28 11:37:37' as data_time,'2022-10-28 11:37:37' as create_time,'2022-10-28 11:37:37' as update_time from null_count\"} },{\"type\":\"hdfs_file\",\"config\":{\"path\":\"hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B
\u8BD5\",\"input_table\":\"null_items\"} }]}"
[INFO] 2022-10-28 11:37:37.091 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[290] - task run command: sudo -u dolphin sh /tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126/17893_46126.command
[INFO] 2022-10-28 11:37:37.093 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[181] - process start, process id is: 3214891
[INFO] 2022-10-28 11:37:37.106 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[205] - process has exited, execute path:/tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126, processId:3214891 ,exitStatusCode:127 ,processWaitForStatus:true ,processExitValue:127
[INFO] 2022-10-28 11:37:38.094 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[63] - -> /tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126/17893_46126.command: line 5: /opt/soft/spark2/bin/spark-submit: No such file or directory
[INFO] 2022-10-28 11:37:38.095 +0800 [taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[57] - FINALIZE_SESSION
@pandong2011
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org