You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by "fanyo190 (via GitHub)" <gi...@apache.org> on 2023/04/12 03:12:27 UTC
[GitHub] [dolphinscheduler] fanyo190 commented on issue #13720: [Bug] [resource center] hive udf always report null
fanyo190 commented on issue #13720:
URL: https://github.com/apache/dolphinscheduler/issues/13720#issuecomment-1504507761
我也遇到了
[LOG-PATH]: /home/dolphinscheduler/worker-server/logs/20230412/9194974006176_1-1242-1318.log, [HOST]: Host{address='192.168.1.174:1234', ip='192.168.1.174', port=1234}
[INFO] 2023-04-12 10:22:20.322 +0800 - Begin to pulling task
[INFO] 2023-04-12 10:22:20.327 +0800 - Begin to initialize task
[INFO] 2023-04-12 10:22:20.327 +0800 - Set task startTime: Wed Apr 12 10:22:20 CST 2023
[INFO] 2023-04-12 10:22:20.327 +0800 - Set task envFile: /home/dolphinscheduler/worker-server/conf/dolphinscheduler_env.sh
[INFO] 2023-04-12 10:22:20.328 +0800 - Set task appId: 1242_1318
[INFO] 2023-04-12 10:22:20.328 +0800 - End initialize task
[INFO] 2023-04-12 10:22:20.328 +0800 - Set task status to TaskExecutionStatus{code=1, desc='running'}
[INFO] 2023-04-12 10:22:20.329 +0800 - TenantCode:hdfs check success
[INFO] 2023-04-12 10:22:20.330 +0800 - ProcessExecDir:/tmp/dolphinscheduler/exec/process/hdfs/7425820897568/9194974006176_1/1242/1318 check success
[INFO] 2023-04-12 10:22:20.330 +0800 - Resources:{} check success
[INFO] 2023-04-12 10:22:20.331 +0800 - Task plugin: SQL create success
[INFO] 2023-04-12 10:22:20.331 +0800 - Success initialized task plugin instance success
[INFO] 2023-04-12 10:22:20.331 +0800 - Success set taskVarPool: null
[INFO] 2023-04-12 10:22:20.331 +0800 - Full sql parameters: SqlParameters{type='HIVE', datasource=15, sql='insert overwrite table dwd.dwd_action_log partition(dt)
select
get_json_object(line,'$.common.ar'),
get_json_object(line,'$.common.ba'),
get_json_object(line,'$.common.ch'),
get_json_object(line,'$.common.md'),
get_json_object(line,'$.common.mid'),
get_json_object(line,'$.common.os'),
get_json_object(line,'$.common.uid'),
get_json_object(line,'$.common.vc'),
get_json_object(line,'$.page.during_time'),
get_json_object(line,'$.page.item'),
get_json_object(line,'$.page.item_type'),
get_json_object(line,'$.page.last_page_id'),
get_json_object(line,'$.page.page_id'),
get_json_object(line,'$.page.sourceType'),
get_json_object(action,'$.action_id'),
get_json_object(action,'$.item'),
get_json_object(action,'$.item_type'),
get_json_object(action,'$.ts'),
'20230409' dt
from ods.ods_log lateral view explode_json_arrays(get_json_object(line,'$.actions')) tmp as action
where get_json_object(line,'$.actions') is not null and from_unixtime(cast(substr(get_json_object(line , '$.ts') ,0, 10) as BIGINT),'yyyyMMdd') = date_format(date_sub(from_unixtime(unix_timestamp(),'yyyy-MM-dd'),1),'yyyyMMdd')', sqlType=1, sendEmail=null, displayRows=10, limit=0, segmentSeparator=, udfs='1', showType='null', connParams='null', groupId='0', title='null', preStatements=[], postStatements=[]}
[INFO] 2023-04-12 10:22:20.331 +0800 - sql type : HIVE, datasource : 15, sql : insert overwrite table dwd.dwd_action_log partition(dt)
select
get_json_object(line,'$.common.ar'),
get_json_object(line,'$.common.ba'),
get_json_object(line,'$.common.ch'),
get_json_object(line,'$.common.md'),
get_json_object(line,'$.common.mid'),
get_json_object(line,'$.common.os'),
get_json_object(line,'$.common.uid'),
get_json_object(line,'$.common.vc'),
get_json_object(line,'$.page.during_time'),
get_json_object(line,'$.page.item'),
get_json_object(line,'$.page.item_type'),
get_json_object(line,'$.page.last_page_id'),
get_json_object(line,'$.page.page_id'),
get_json_object(line,'$.page.sourceType'),
get_json_object(action,'$.action_id'),
get_json_object(action,'$.item'),
get_json_object(action,'$.item_type'),
get_json_object(action,'$.ts'),
'20230409' dt
from ods.ods_log lateral view explode_json_arrays(get_json_object(line,'$.actions')) tmp as action
where get_json_object(line,'$.actions') is not null and from_unixtime(cast(substr(get_json_object(line , '$.ts') ,0, 10) as BIGINT),'yyyyMMdd') = date_format(date_sub(from_unixtime(unix_timestamp(),'yyyy-MM-dd'),1),'yyyyMMdd') , localParams : [],udfs : 1,showType : null,connParams : null,varPool : [] ,query max result limit 0
[ERROR] 2023-04-12 10:22:20.332 +0800 - sql task error
java.lang.NullPointerException: null
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.lambda$buildJarSql$1(SqlTask.java:500)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.buildJarSql(SqlTask.java:505)
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.createFuncs(SqlTask.java:474)
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.handle(SqlTask.java:158)
at org.apache.dolphinscheduler.server.worker.runner.DefaultWorkerDelayTaskExecuteRunnable.executeTask(DefaultWorkerDelayTaskExecuteRunnable.java:49)
at org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable.run(WorkerTaskExecuteRunnable.java:174)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:74)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
[ERROR] 2023-04-12 10:22:20.333 +0800 - Task execute failed, due to meet an exception
org.apache.dolphinscheduler.plugin.task.api.TaskException: Execute sql task failed
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.handle(SqlTask.java:168)
at org.apache.dolphinscheduler.server.worker.runner.DefaultWorkerDelayTaskExecuteRunnable.executeTask(DefaultWorkerDelayTaskExecuteRunnable.java:49)
at org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable.run(WorkerTaskExecuteRunnable.java:174)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:74)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NullPointerException: null
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.lambda$buildJarSql$1(SqlTask.java:500)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.buildJarSql(SqlTask.java:505)
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.createFuncs(SqlTask.java:474)
at org.apache.dolphinscheduler.plugin.task.sql.SqlTask.handle(SqlTask.java:158)
... 9 common frames omitted
[INFO] 2023-04-12 10:22:20.334 +0800 - Get a exception when execute the task, will send the task execute result to master, the current task execute result is TaskExecutionStatus{code=6, desc='failure'}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org