You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by li...@apache.org on 2021/01/11 06:54:46 UTC
[incubator-dolphinscheduler] branch master updated: [Merge] merge
from 1.3.4 (#4371)
This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/master by this push:
new 0cf173f [Merge] merge from 1.3.4 (#4371)
0cf173f is described below
commit 0cf173f37bdae4a674d7b3e64cf7a941728a57ee
Author: lgcareer <18...@163.com>
AuthorDate: Mon Jan 11 14:54:35 2021 +0800
[Merge] merge from 1.3.4 (#4371)
* fix
* [fix-3058][ui]Move rtTargetArr to jsPlumbHandle.js
* [optimization][ui]Prevent the shell script input box from being empty
* [Fix-3462][api]If login user is admin,need list all udfs (#3465)
* [Fix-3462][api]If login user is admin,need list all udfs
* [Fix-3462][api]add the test on the method of QueryUdfFuncList
* [Fix-3462][api]fix the code smell
* [Fix-3462][api]fix the code smell
* [Fix-3462][api]fix the code smell
* [Fix-3463][api]Fixed that run the sql task will be failure after rename the udf resource (#3482)
* [fixBug-3058][ui]Fix connection abnormalities in historical workflow instance data
* [Feture-3327][ui]Add the function of re-uploading files in the udf subdirectory
* fix bug: Fix master task dependency check bug (#3473)
Co-authored-by: lenboo <ba...@analysys.com.cn>
* [maven-release-plugin] prepare release 1.3.2
* [maven-release-plugin] prepare for next development iteration
* fix ci_e2e fail (#3497)
* [Fix-3469][api]Should filter the resource by the different program type (#3498)
* [Fix-3463][api]Fixed that run the sql task will be failure after rename the udf resource
* [Fix-3469][api]Should list python file and jar file
* [Fix-3469][api]Should filter the resource by the different program type
* [Fix-3469][api]fix the code smell
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback (#3499)
* [Feature] JVM parameter optimization , related issue #3370
* [Feature] JVM parameter optimization , related issue #3370
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* [maven-release-plugin] prepare release 1.3.2
* [maven-release-plugin] prepare for next development iteration
* [Fix-3469][ui]The value of maintenance resources and the filtering of resources according to different program types
* fix
* Revert "fix ci_e2e fail (#3497)"
This reverts commit e367f90bb73c9682739308a0a98887a1c0f407ef.
* test
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback (#3503)
* [Feature] JVM parameter optimization , related issue #3370
* [Feature] JVM parameter optimization , related issue #3370
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback
* test
* test release 1.3.2 version rollback
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* [maven-release-plugin] prepare release 1.3.2
* [maven-release-plugin] prepare for next development iteration
* test release 1.3.2 version rollback (#3504)
* [Feature] JVM parameter optimization , related issue #3370
* [Feature] JVM parameter optimization , related issue #3370
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback
* test
* test release 1.3.2 version rollback
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* [maven-release-plugin] prepare release 1.3.2
* [maven-release-plugin] prepare for next development iteration
* fix ds muti-level directory in zk, which lead to fail to assign work
* add login user check some actions in api
* [Hotfix][ci] Fix e2e ci docker image build error
* modify tag 1.3.0 to HEAD
* modify tag 1.3.0 to HEAD (#3525)
* [Feature] JVM parameter optimization , related issue #3370
* [Feature] JVM parameter optimization , related issue #3370
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback
* test
* test release 1.3.2 version rollback
* modify tag 1.3.0 to HEAD
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* remove OGNL part of the mybaits notice (#3526)
* [maven-release-plugin] prepare release 1.3.2
* [maven-release-plugin] prepare for next development iteration
* release 1.3.2 version rollback (#3527)
* [Feature] JVM parameter optimization , related issue #3370
* [Feature] JVM parameter optimization , related issue #3370
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback
* test
* test release 1.3.2 version rollback
* modify tag 1.3.0 to HEAD
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* [ui]Script input box to modify the delay loading time
* fix
* fix
* fix
* fix
* modify general user can't create token
* [ui]It is forbidden to select non-existent resources and modify the tree display data format
* modify general user can't create token (#3533)
* [Feature] JVM parameter optimization , related issue #3370
* [Feature] JVM parameter optimization , related issue #3370
* test release 1.3.2 version rollback
* test release 1.3.2 version rollback
* test
* test release 1.3.2 version rollback
* modify tag 1.3.0 to HEAD
* modify general user can't create token
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* if task is null , set task type is null instead of "null"
* [Fix-3536][api]If user didn't have tenant,create resource directory will NPE (#3537)
* [Fix-3536][api]If user didn't have tenant,create resource will NPE
* [Fix-3536][api]If user didn't have tenant,create resource directory will NPE
* modify general user can't create,delete,update token (#3538)
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* [ui]Resource delete OK button to increase loading, change the number of homepage display cursor
* fix
* [Fix-3616][Server] when worker akc/response master exception , async retry (#3748)
* [fixbug][ui]Repair the master and worker management instrument display
* [Fix-3238][docker]Fix that can not create folder in docker with standalone mode (#3741)
* [fixbug][ui]Remove non-existent or deleted resources disabled
* [fixBug-3621][ui]If the workflow instance status is executing status, it is forbidden to select
* [fix-3553][ui]Repair click workflow connection, select the entire path
* fix
* fix
* [Fix-3238][docker]Fix that can not create folder in docker with standalone mode
* [Fix-3616][Server] when worker akc/response master exception , async retry (#3776)
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
* The batch delete function in the workflow definition and workflow instance pages cannot be canceled if selected.
* [Improvement-3720][ui] js mailbox verification fix
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect (#3784)
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* The batch delete function in the workflow definition and workflow instance pages cannot be canceled if selected.
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
Co-authored-by: zhuangchong <zh...@163.com>
Co-authored-by: JinyLeeChina <42...@users.noreply.github.com>
* [Fix-3124][docker]Fix that can not build a docker image on windows (#3765)
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect (#3786)
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* The batch delete function in the workflow definition and workflow instance pages cannot be canceled if selected.
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
Co-authored-by: zhuangchong <zh...@163.com>
Co-authored-by: JinyLeeChina <42...@users.noreply.github.com>
* [Fix-3258][Security][Worker group manage] Connot get create time and update time,report DateTimeParseException (#3787)
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* [Fix-3616][Server] when worker akc/response master exception , async retry
* The batch delete function in the workflow definition and workflow instance pages cannot be canceled if selected.
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
* [Fix-3549] [Server][sqlTask]The alias column in the query SQL does not take effect
* [BugFixed] issue #3258 (#3265)
* 'ExecutionStatus'
* '3258'
* Update WorkerGroupServiceTest.java
* Delete UserState.java
* Delete ResourceSyncService.java
* Delete core-site.xml
* Delete hdfs-site.xml
Co-authored-by: dailidong <da...@gmail.com>
Co-authored-by: qiaozhanwei <qi...@outlook.com>
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
Co-authored-by: zhuangchong <zh...@163.com>
Co-authored-by: JinyLeeChina <42...@users.noreply.github.com>
Co-authored-by: dailidong <da...@gmail.com>
* [fixBug-3792][ui]Click on the sidebar to adapt the width of the pie chart on the project homepage
* [Bug-3713][HadoopUtils] catfile method Stream not closed (#3715)
* fix bug
Delete invalid field: executorcores
Modify verification prompt
* fix bug
Delete invalid field: executorcores
Modify verification prompt
* fix bug
Delete invalid field: executorcores
Modify verification prompt
* dag add close button
* reset last version
* reset last version
* dag add close buttion
dag add close buttion
* update CLICK_SAVE_WORKFLOW_BUTTON xpath
* updae CLICK_SAVE_WORKFLOW_BUTTON xpath
* updae CLICK_SAVE_WORKFLOW_BUTTON xpath
* updae CLICK_SAVE_WORKFLOW_BUTTON xpath
* Update CreateWorkflowLocator.java
modify submit workflow button
* Update CreateWorkflowLocator.java
* Update CreateWorkflowLocator.java
modify CLICK_ADD_BUTTON
* Update CreateWorkflowLocator.java
delete print
* Update CreateWorkflowLocator.java
1
* Update CreateWorkflowLocator.java
1
* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks
in addition
update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'
* Update dag.vue
* Update dag.vue
* Update dag.vue
* Update CreateWorkflowLocator.java
* Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks"
This reverts commit 3a2cba7a
* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks
in addition
update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'
* Update dolphinscheduler-daemon.sh
* catfile method Stream not closed
* catfile method Stream not closed
Co-authored-by: dailidong <da...@gmail.com>
Co-authored-by: xingchun-chen <55...@users.noreply.github.com>
* [Fix-#3713][common]Fix that catfile method Stream not closed
* [Fix-#3713][common]Fix that catfile method Stream not closed (#3810)
* [Bug-3713][HadoopUtils] catfile method Stream not closed (#3715)
* fix bug
Delete invalid field: executorcores
Modify verification prompt
* fix bug
Delete invalid field: executorcores
Modify verification prompt
* fix bug
Delete invalid field: executorcores
Modify verification prompt
* dag add close button
* reset last version
* reset last version
* dag add close buttion
dag add close buttion
* update CLICK_SAVE_WORKFLOW_BUTTON xpath
* updae CLICK_SAVE_WORKFLOW_BUTTON xpath
* updae CLICK_SAVE_WORKFLOW_BUTTON xpath
* updae CLICK_SAVE_WORKFLOW_BUTTON xpath
* Update CreateWorkflowLocator.java
modify submit workflow button
* Update CreateWorkflowLocator.java
* Update CreateWorkflowLocator.java
modify CLICK_ADD_BUTTON
* Update CreateWorkflowLocator.java
delete print
* Update CreateWorkflowLocator.java
1
* Update CreateWorkflowLocator.java
1
* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks
in addition
update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'
* Update dag.vue
* Update dag.vue
* Update dag.vue
* Update CreateWorkflowLocator.java
* Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks"
This reverts commit 3a2cba7a
* Setting '-XX:+DisableExplicitGC ' causes netty memory leaks
in addition
update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M'
* Update dolphinscheduler-daemon.sh
* catfile method Stream not closed
* catfile method Stream not closed
Co-authored-by: dailidong <da...@gmail.com>
Co-authored-by: xingchun-chen <55...@users.noreply.github.com>
* [Fix-#3713][common]Fix that catfile method Stream not closed
Co-authored-by: BoYiZhang <39...@users.noreply.github.com>
Co-authored-by: dailidong <da...@gmail.com>
Co-authored-by: xingchun-chen <55...@users.noreply.github.com>
* [Fix-#3487][api、dao] cherry pick from dev to fix that create folder duplicate name under multithreading
* [Hotfix-3131][api] Fix the new tenant already exists prompt (#3132)
* Bugfix: Fix the new tenant already exists prompt
* Feature: Add test cases
* Update TenantServiceTest.java
Co-authored-by: dailidong <da...@gmail.com>
Co-authored-by: qiaozhanwei <qi...@outlook.com>
* Set up JDK 11 for SonarCloud in github action. (#3052)
* Set up JDK 11 for SonarCloud in github action.
* Fix javadoc error with JDK 11.
* Prevent Javadoc from stopping if it finds any html errors.
* [fixBug-3621][ui]Select the batch checkbox to unfilter the instances in the executing state
* add verify tenant name cannot contain special characters.
* [fixBug-3840][ui]The tenant code only allows letters or a combination of letters and numbers
* fix
* fix
* fix
* [Fix-#3702][api] When re-upload the resource file but don't change the name or desc,it need replace the origin resource file. (#3862)
* [Fix-#3702][api] When re-upload the resource file but don't change the name or desc,it will not replace the origin resource file.
* [Fix-#3702][api] When re-upload the resource file but don't change the name or desc,it will not replace the origin resource file.
* [fixbug-3621][ui]Workflow instance ready to stop and ready to suspend state prohibits checking
* [fixbug-3887][ui]Fix missing English translation of re-upload files
* add process define name verify. (#3879)
* Revert "[1.3.3-release][fix-3835][ui] When the tenantName contains "<", the tenant drop-down list is blankadd verify tenant name cannot contain special characters."
* revert pr 3872
* [FIX-3617][Service]after subtask fault tolerance, 2 task instances are generated (#3830)
* fix bug(#3617): after subtask fault tolerance, 2 task instances are generated.
* delete unused code
* update code smell
* refactor sub work command process
* add process service ut
* add license header
* fix some code smell
* chang ut java8 to java11
* update sonar to java11
* copy ut config from dev
* remove checkstyle
* revert to 1.3.3
* change proess service test to executor service
* add process service test
* add process service test
* revert
* revert
* add comments
* change dev to 1.3.3-release
* revert
Co-authored-by: baoliang <ba...@analysys.com.cn>
* [Fix-#3487][sql] add dolphinscheduler_dml.sql under 1.3.3_schema (#3907)
* [FIX-3836][1.3.3-release-API] process definition validation name interface prompt information error (#3899)
* fix bug : error message
* fix code smell
* fix code smell
* [FIX_#3789][remote]cherry pick from dev to support netty heart beat
* [FIX_#3789][remote]cherry pick from dev to support netty heart beat
* [FIX_#3789][remote]cherry pick from dev to support netty heart beat (#3913)
* [FIX_#3789][remote]cherry pick from dev to support netty heart beat
* [FIX_#3789][remote]cherry pick from dev to support netty heart beat
Co-authored-by: Kirs <ac...@163.com>
* Repair check box cannot be canceled
* [fix-3843][api] When update workflow definition,if name already exists, the prompt is not friendly
* [fix-3843][api] When update workflow definition,if name already exists, the prompt is not friendly
* [fix-#3843][api]When update workflow definition,if name already exists, the prompt is not friendly (#3918)
* [FIX_#3789][remote]cherry pick from dev to support netty heart beat
* [FIX_#3789][remote]cherry pick from dev to support netty heart beat
* [fix-3843][api] When update workflow definition,if name already exists, the prompt is not friendly
* [fix-3843][api] When update workflow definition,if name already exists, the prompt is not friendly
Co-authored-by: Kirs <ac...@163.com>
* [Fix-#3487][sql] update uc_dolphin_T_t_ds_resources_un
* Workflow definition name re-modified and added check
* [fix-#3843][api]When update workflow definition,if name already exists, the prompt is not friendly.
* update code.
* [#3931][ui]Field name optimization for spark, flink, and mr
* change version from 1.3.2-SNAPSHOT to 1.3.3-SNAPSHOT (#3934)
* [maven-release-plugin] prepare release 1.3.3
* [maven-release-plugin] prepare for next development iteration
* [ambari-plugin]change version 1.3.2 to 1.3.3 (#3935)
* fix bug:3615 After the task is executed successfully, but the next task has not been submitted, stop the master
* [fixBug-3964][ui]Switch back and forth over timeout alarm, the selected value is empty
* solve too many files, close logClientService (#3971)
* fix #3966 sub process doesnot send alert mail after process instance ending. (#3972)
Co-authored-by: baoliang <ba...@analysys.com.cn>
* [Fix-#3618][server] resolve task executed finished but not release the file handle (#3975)
* [Fix-#3618][server] resolve task executed finished but not release the file handle
* [Fix-#3618][server] resolve task executed finished but not release the file handle
* [Fix-#3958][api] files should not be created successfully in the directory of the authorized file
* [FIX-3966] The timeout warning does not take effect in sub_process (#3982)
* fix #3966 sub process doesnot send alert mail after process instance ending.
* fix bug 3964: sub_process The timeout warning does not take effect
add timeout warning for sub_process/dependent task.
* fix code smell
* fix code smell
* fix code smell
* update worker group inherit from parent
Co-authored-by: baoliang <ba...@analysys.com.cn>
* fix import dolphinscheduler_mysql.sql insert admin user data
* [FIX-3929] condition task would post wrong tasks when failover. (#3999)
* fix #3966 sub process doesnot send alert mail after process instance ending.
* fix bug 3964: sub_process The timeout warning does not take effect
add timeout warning for sub_process/dependent task.
* fix code smell
* fix code smell
* fix code smell
* update worker group inherit from parent
* remove stdout in logback configuration
* fix bug #3929 condition task would post error when failover.
* remove unused test
* add comments
* add skip node judge
Co-authored-by: baoliang <ba...@analysys.com.cn>
* [FIX-3929] because of no lock, start up failover would dispatch two same tasks. (#4004)
* fix #3966 sub process doesnot send alert mail after process instance ending.
* fix bug 3964: sub_process The timeout warning does not take effect
add timeout warning for sub_process/dependent task.
* fix code smell
* fix code smell
* fix code smell
* update worker group inherit from parent
* remove stdout in logback configuration
* fix bug #3929 condition task would post error when failover.
* remove unused test
* add comments
* add skip node judge
* fix bug 3929: because of no lock, start up failover would dispatch two same tasks.
Co-authored-by: baoliang <ba...@analysys.com.cn>
* revert pom version to 1.3.3-release
* [maven-release-plugin] prepare release 1.3.3
* [maven-release-plugin] prepare for next development iteration
* [release]revert pom version to 1.3.3-release
* fix bug 4010: remove failed condition tasks from error-task-list. (#4011)
Co-authored-by: baoliang <ba...@analysys.com.cn>
* [maven-release-plugin] prepare release 1.3.3
* [maven-release-plugin] prepare for next development iteration
* [Fix-4054][Api] Fix The last week of the month for adding/editing timing, preview and save timing will report an error
* [FIX#4033]cherry pick from dev to resolve that `$[]`` conflicts with mysql keywords (#4142)
* [FIX#4033]cherry pick from dev to resolve that `$[]`` conflicts with mysql keywords
* [FIX#4033]cherry pick from dev to resolve that `$[]`` conflicts with mysql keywords
* [FIX#4033]cherry pick from dev to resolve that `$[]`` conflicts with mysql keywords
Co-authored-by: Kirs <ac...@163.com>
* [Fix][Server] Fix clear task execute path is related to master.
* [update] Fix code smells
* [update] add unit test.
* fix ut
* code style
* code style
* code style
* sqoop task optimization
* sqoop front-end optimization
* modify sqoop task UT
* add sqoop task UT to pom
* sqoop task source type or target type is null throw exception
* fix testSqoopTaskTest bug (#3024)
* [FIX-4034][server] fix sqoop import fail (#4036)
* fix #4043, sqoop import query fail
* fix #4043, sqoop task hard code & code style
* add license for SqoopConstants
* add private constructor for SqoopConstants
* fixed sqoop mysql pwd have special character
* fix checkstyle
* fix sqoop task log
* remove unused constants
* [FIX-4034][server] fix sqoop import fail (#4036)
* fix #4043, sqoop import query fail
* fix #4043, sqoop task hard code & code style
* add license for SqoopConstants
* add private constructor for SqoopConstants
* fixed sqoop mysql pwd have special character
* fix checkstyle
* fix sqoop task log
* remove unused constants
* fix sqoop task jdbc string contains special char (#4105)
* split sqoop import hive database and table (#4141)
* modify JSONUtils
* Merge pull request #4088 from liliang1991/dev11_20 (#4154)
[Fix-4090][dao] Add null judgment
Co-authored-by: dailidong <da...@gmail.com>
* [FIX-3900][server] Cherry pick from dev to kill multi yarn app in one job (#4151)
* [FIX-3900][server] Cherry pick from dev to kill multi yarn app in one job
* [FIX-3900][server] Cherry pick from dev to kill multi yarn app in one job
Co-authored-by: Eights-LI <ye...@gmail.com>
* [Feature-2925][server] Cherry pick from dev to init TaskLogger in TaskExecuteProcessor (#4163)
* [Feature-2925][server] Cherry pick from dev to init TaskLogger in TaskExecuteProcessor
* [Feature-2925][server] Cherry pick from dev to init TaskLogger in TaskExecuteProcessor
Co-authored-by: Yichao Yang <10...@qq.com>
* [FIX-#4084][server]cherry pick from dev to fix taskInstance state change error (#4171)
Co-authored-by: CalvinKirs <ac...@163.com>
* [Fix-4143][api,service][jar upgrade]cherry from dev to upgrade quartz version to 2.3.0 (#4173)
* [Fix-4143][api,service][jar upgrade]cherry from dev to upgrade quartz version to 2.3.0
* [Fix-4143][api,service][jar upgrade]cherry from dev to upgrade quartz version to 2.3.0
* [Fix-4143][api,service][jar upgrade]cherry from dev to upgrade quartz version to 2.3.0
* [Fix-4143][api,service][jar upgrade]cherry from dev to upgrade quartz version to 2.3.0
* [Fix-4143][api,service][jar upgrade]cherry from dev to upgrade quartz version to 2.3.0
Co-authored-by: dailidong <da...@gmail.com>
* chrry-pick sqoop task in dev to 1.3.4 (#4168)
* sqoop task optimization
* sqoop front-end optimization
* modify sqoop task UT
* add sqoop task UT to pom
* sqoop task source type or target type is null throw exception
* fix testSqoopTaskTest bug (#3024)
* [FIX-4034][server] fix sqoop import fail (#4036)
* fix #4043, sqoop import query fail
* fix #4043, sqoop task hard code & code style
* add license for SqoopConstants
* add private constructor for SqoopConstants
* fixed sqoop mysql pwd have special character
* fix checkstyle
* fix sqoop task log
* remove unused constants
* [FIX-4034][server] fix sqoop import fail (#4036)
* fix #4043, sqoop import query fail
* fix #4043, sqoop task hard code & code style
* add license for SqoopConstants
* add private constructor for SqoopConstants
* fixed sqoop mysql pwd have special character
* fix checkstyle
* fix sqoop task log
* remove unused constants
* fix sqoop task jdbc string contains special char (#4105)
* split sqoop import hive database and table (#4141)
* modify JSONUtils
Co-authored-by: CalvinKirs <ac...@163.com>
* cancel sqoop sql query import hive limit
* [Improve-4143][docker]Some parameters were improved for Docker
* cherry-pick load TaskInstance slow #4184
* [Improve-4143][docker]Some parameters were improved for Docker
* update sqoop task params when upgrade to 1.3.4
* Clear code smells.
* [Improve-4143][docker]No need to install kazoo
* [Improve-4143][docker]Update the docs how to build and run with docker
* cherry-pick task_ack_miss (#4198)
* fix 4190: When the amount of json data is large, process list page display slowly
* [PrepareRelease][Ambari][SOFT_VERSION]Change version from 1.3.3 to 1.3.4 (#4203)
* sqoop hive target add target dir field
* add target-dir param in sqoop task hive TargetHiveParameter
* [Feature-3878][UI]Repair the last Sunday of each month
* [Feature-4138][Master]Cherry pick from dev to dispatch add sleep when dispatch task to work error (#4211)
* [Feature-4138][Master]Cherry pick from dev to dispatch add sleep when dispatch task to work error
* [Feature-4138][Master]Cherry pick from dev to dispatch add sleep when dispatch task to work error
Co-authored-by: BoYiZhang <39...@users.noreply.github.com>
* #4182-cherry-pick (#4210)
* cherry pick #4219 to 1.3.4-pre (#4220)
* cherry pick #4288 to 1.3.4-pre (#4229)
* cherry pick #4288 to 1.3.4-pre
* remove unused import
* [cherry-pick to 1.3.4-pre][ui] Fix dag node name verification
* [Fix-4222][Master]Add the priority queue to ensure that tasks are submitted according to priority.
* [Fix-4222][Master]Add the priority queue to ensure that tasks are submitted according to priority.
* [Fix-4222][Master]update print log.
* cherry pick #4288 to 1.3.4-prepare
* cherry pick date param cover
* fix time cover
* fix time cover
* fix time cover
* fix time cover
* [maven-release-plugin] prepare release 1.3.4
* [maven-release-plugin] prepare for next development iteration
Co-authored-by: break60 <79...@qq.com>
Co-authored-by: xingchun-chen <55...@users.noreply.github.com>
Co-authored-by: bao liang <29...@users.noreply.github.com>
Co-authored-by: lenboo <ba...@analysys.com.cn>
Co-authored-by: qiaozhanwei <qi...@analysys.com.cn>
Co-authored-by: qiaozhanwei <qi...@outlook.com>
Co-authored-by: XiaotaoYi <v-...@hotmail.com>
Co-authored-by: Yichao Yang <10...@qq.com>
Co-authored-by: zhuangchong <zh...@163.com>
Co-authored-by: JinyLeeChina <42...@users.noreply.github.com>
Co-authored-by: dailidong <da...@gmail.com>
Co-authored-by: BoYiZhang <39...@users.noreply.github.com>
Co-authored-by: muzhongjiang <mu...@163.com>
Co-authored-by: Jave-Chen <ba...@gmail.com>
Co-authored-by: zhuangchong <zh...@163.com>
Co-authored-by: zhuangchong <37...@users.noreply.github.com>
Co-authored-by: Kirs <ac...@163.com>
Co-authored-by: lgcareer <lg...@apache.org>
Co-authored-by: wulingqi <wu...@baijiahulian.com>
Co-authored-by: yangquan <iy...@qq.com>
Co-authored-by: yh2388 <17...@qq.com>
Co-authored-by: felix.Wang <fe...@thinkingdata.cn>
Co-authored-by: eights <am...@my.com>
Co-authored-by: Yelli <ye...@gmail.com>
Co-authored-by: wangxj3 <85...@qq.com>
Co-authored-by: lenboo <ba...@gmail.com>
Co-authored-by: lidongdai <li...@apache.org>
---
.../common-services/DOLPHIN/1.3.4/alerts.json | 158 ++++
.../DOLPHIN/1.3.4/configuration/dolphin-alert.xml | 137 +++
.../configuration/dolphin-application-api.xml | 87 ++
.../DOLPHIN/1.3.4/configuration/dolphin-common.xml | 158 ++++
.../1.3.4/configuration/dolphin-datasource.xml | 206 +++++
.../DOLPHIN/1.3.4/configuration/dolphin-env.xml | 123 +++
.../DOLPHIN/1.3.4/configuration/dolphin-master.xml | 88 ++
.../DOLPHIN/1.3.4/configuration/dolphin-quartz.xml | 126 +++
.../DOLPHIN/1.3.4/configuration/dolphin-worker.xml | 67 ++
.../1.3.4/configuration/dolphin-zookeeper.xml | 76 ++
.../common-services/DOLPHIN/1.3.4/metainfo.xml | 137 +++
.../alerts/alert_dolphin_scheduler_status.py | 124 +++
.../1.3.4/package/scripts/dolphin_alert_service.py | 62 ++
.../1.3.4/package/scripts/dolphin_api_service.py | 71 ++
.../DOLPHIN/1.3.4/package/scripts/dolphin_env.py | 123 +++
.../package/scripts/dolphin_logger_service.py | 61 ++
.../package/scripts/dolphin_master_service.py | 62 ++
.../package/scripts/dolphin_worker_service.py | 61 ++
.../DOLPHIN/1.3.4/package/scripts/params.py | 155 ++++
.../DOLPHIN/1.3.4/package/scripts/service_check.py | 31 +
.../DOLPHIN/1.3.4/package/scripts/status_params.py | 23 +
.../1.3.4/package/templates/alert.properties.j2 | 15 +-
.../templates/application-api.properties.j2 | 15 +-
.../1.3.4/package/templates/common.properties.j2 | 15 +-
.../package/templates/datasource.properties.j2 | 15 +-
.../1.3.4/package/templates/dolphin-daemon.sh.j2 | 116 +++
.../1.3.4/package/templates/master.properties.j2 | 15 +-
.../1.3.4/package/templates/quartz.properties.j2 | 15 +-
.../1.3.4/package/templates/worker.properties.j2 | 15 +-
.../package/templates/zookeeper.properties.j2 | 15 +-
.../DOLPHIN/1.3.4/quicklinks/quicklinks.json | 26 +
.../DOLPHIN/1.3.4/themes/theme.json | 661 ++++++++++++++
ambari_plugin/statcks/DOLPHIN/metainfo.xml | 2 +-
docker/build/Dockerfile | 3 +-
docker/build/README.md | 68 +-
docker/build/README_zh_CN.md | 66 +-
.../conf/dolphinscheduler/common.properties.tpl | 8 +-
.../dolphinscheduler/datasource.properties.tpl | 15 +-
.../conf/dolphinscheduler/zookeeper.properties.tpl | 2 +-
docker/build/startup-init-conf.sh | 20 +-
docker/build/startup.sh | 40 +-
docker/docker-swarm/docker-compose.yml | 43 +-
docker/docker-swarm/docker-stack.yml | 40 +-
dolphinscheduler-alert/pom.xml | 2 +-
dolphinscheduler-api/pom.xml | 30 +-
.../apache/dolphinscheduler/api/enums/Status.java | 2 +-
dolphinscheduler-common/pom.xml | 2 +-
.../apache/dolphinscheduler/common/Constants.java | 24 +
.../common/enums/SqoopJobType.java | 43 +-
.../common/enums/SqoopQueryType.java | 53 +-
.../common/shell/AbstractShell.java | 38 +-
.../common/task/sqoop/SqoopParameters.java | 99 ++-
.../task/sqoop/targets/TargetHiveParameter.java | 13 +
.../task/sqoop/targets/TargetMysqlParameter.java | 2 +-
.../dolphinscheduler/common/utils/FileUtils.java | 60 +-
.../dolphinscheduler/common/utils/LoggerUtils.java | 21 +
.../dolphinscheduler/common/utils/OSUtils.java | 87 +-
.../common/utils/ParameterUtils.java | 418 ++++-----
.../dolphinscheduler/common/utils/StringUtils.java | 22 +-
.../utils/placeholder/TimePlaceholderUtils.java | 61 +-
.../dolphinscheduler/common/utils/OSUtilsTest.java | 6 +-
.../common/utils/ParameterUtilsTest.java | 84 +-
.../placeholder/TimePlaceholderUtilsTest.java | 76 +-
dolphinscheduler-dao/pom.xml | 2 +-
.../dao/upgrade/DolphinSchedulerManager.java | 2 +
.../dolphinscheduler/dao/upgrade/UpgradeDao.java | 46 +
.../dolphinscheduler/dao/utils/DagHelper.java | 9 +-
.../dao/mapper/ProcessInstanceMapper.xml | 7 +-
.../dao/mapper/TaskInstanceMapper.xml | 4 +-
dolphinscheduler-dist/pom.xml | 2 +-
dolphinscheduler-dist/release-docs/LICENSE | 11 +-
dolphinscheduler-plugin-api/pom.xml | 2 +-
dolphinscheduler-remote/pom.xml | 2 +-
dolphinscheduler-server/pom.xml | 2 +-
.../dolphinscheduler/server/log/TaskLogFilter.java | 9 +-
.../master/consumer/TaskPriorityQueueConsumer.java | 60 +-
.../processor/queue/TaskResponseService.java | 17 +-
.../server/master/runner/MasterExecThread.java | 90 +-
.../server/utils/ProcessUtils.java | 714 ++++++++-------
.../cache/TaskExecutionContextCacheManager.java | 9 +
.../impl/TaskExecutionContextCacheManagerImpl.java | 7 +
.../worker/processor/TaskExecuteProcessor.java | 67 +-
.../server/worker/processor/TaskKillProcessor.java | 12 +-
.../server/worker/runner/TaskExecuteThread.java | 113 ++-
.../worker/task/AbstractCommandExecutor.java | 18 +-
.../server/worker/task/shell/ShellTask.java | 36 +-
.../server/worker/task/sqoop/SqoopConstants.java | 75 ++
.../server/worker/task/sqoop/SqoopTask.java | 50 +-
.../task/sqoop/generator/CommonGenerator.java | 63 +-
.../task/sqoop/generator/ISourceGenerator.java | 6 +-
.../task/sqoop/generator/ITargetGenerator.java | 6 +-
.../task/sqoop/generator/SqoopJobGenerator.java | 54 +-
.../generator/sources/HdfsSourceGenerator.java | 36 +-
.../generator/sources/HiveSourceGenerator.java | 47 +-
.../generator/sources/MysqlSourceGenerator.java | 120 +--
.../generator/targets/HdfsTargetGenerator.java | 56 +-
.../generator/targets/HiveTargetGenerator.java | 72 +-
.../generator/targets/MysqlTargetGenerator.java | 88 +-
.../consumer/TaskPriorityQueueConsumerTest.java | 456 +++++++++-
.../server/utils/ProcessUtilsTest.java | 111 ++-
.../worker/runner/TaskExecuteThreadTest.java | 126 +++
.../server/worker/task/sqoop/SqoopTaskTest.java | 185 ++--
dolphinscheduler-service/pom.xml | 18 +-
.../queue/PeerTaskInstancePriorityQueue.java | 124 +++
.../service/queue/TaskPriorityQueue.java | 6 +-
.../service/queue/TaskPriorityQueueImpl.java | 7 +-
dolphinscheduler-ui/pom.xml | 2 +-
.../home/pages/dag/_source/formModel/formModel.vue | 9 +-
.../_source/formModel/tasks/_source/datasource.vue | 2 +-
.../pages/dag/_source/formModel/tasks/sqoop.vue | 966 +++++++++++++--------
.../components/crontab/source/_times/day.vue | 8 +-
.../module/components/crontab/source/util/index.js | 2 +-
.../src/js/module/i18n/locale/en_US.js | 9 +
.../src/js/module/i18n/locale/zh_CN.js | 9 +
pom.xml | 9 +-
sql/soft_version | 2 +-
.../1.3.4_schema/mysql/dolphinscheduler_ddl.sql | 29 +-
.../1.3.4_schema/mysql/dolphinscheduler_dml.sql | 30 +-
.../postgresql/dolphinscheduler_ddl.sql | 29 +-
.../postgresql/dolphinscheduler_dml.sql | 30 +-
tools/dependencies/known-dependencies.txt | 10 +-
121 files changed, 6578 insertions(+), 2024 deletions(-)
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/alerts.json b/ambari_plugin/common-services/DOLPHIN/1.3.4/alerts.json
new file mode 100644
index 0000000..08c7b22
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/alerts.json
@@ -0,0 +1,158 @@
+{
+ "DOLPHIN": {
+ "service": [],
+ "DOLPHIN_API": [
+ {
+ "name": "dolphin_api_port_check",
+ "label": "dolphin_api_port_check",
+ "description": "dolphin_api_port_check.",
+ "interval": 10,
+ "scope": "ANY",
+ "source": {
+ "type": "PORT",
+ "uri": "{{dolphin-application-api/server.port}}",
+ "default_port": 12345,
+ "reporting": {
+ "ok": {
+ "text": "TCP OK - {0:.3f}s response on port {1}"
+ },
+ "warning": {
+ "text": "TCP OK - {0:.3f}s response on port {1}",
+ "value": 1.5
+ },
+ "critical": {
+ "text": "Connection failed: {0} to {1}:{2}",
+ "value": 5.0
+ }
+ }
+ }
+ }
+ ],
+ "DOLPHIN_LOGGER": [
+ {
+ "name": "dolphin_logger_port_check",
+ "label": "dolphin_logger_port_check",
+ "description": "dolphin_logger_port_check.",
+ "interval": 10,
+ "scope": "ANY",
+ "source": {
+ "type": "PORT",
+ "uri": "{{dolphin-common/loggerserver.rpc.port}}",
+ "default_port": 50051,
+ "reporting": {
+ "ok": {
+ "text": "TCP OK - {0:.3f}s response on port {1}"
+ },
+ "warning": {
+ "text": "TCP OK - {0:.3f}s response on port {1}",
+ "value": 1.5
+ },
+ "critical": {
+ "text": "Connection failed: {0} to {1}:{2}",
+ "value": 5.0
+ }
+ }
+ }
+ }
+ ],
+ "DOLPHIN_MASTER": [
+ {
+ "name": "DOLPHIN_MASTER_CHECK",
+ "label": "check dolphin scheduler master status",
+ "description": "",
+ "interval":10,
+ "scope": "HOST",
+ "enabled": true,
+ "source": {
+ "type": "SCRIPT",
+ "path": "DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py",
+ "parameters": [
+
+ {
+ "name": "connection.timeout",
+ "display_name": "Connection Timeout",
+ "value": 5.0,
+ "type": "NUMERIC",
+ "description": "The maximum time before this alert is considered to be CRITICAL",
+ "units": "seconds",
+ "threshold": "CRITICAL"
+ },
+ {
+ "name": "alertName",
+ "display_name": "alertName",
+ "value": "DOLPHIN_MASTER",
+ "type": "STRING",
+ "description": "alert name"
+ }
+ ]
+ }
+ }
+ ],
+ "DOLPHIN_WORKER": [
+ {
+ "name": "DOLPHIN_WORKER_CHECK",
+ "label": "check dolphin scheduler worker status",
+ "description": "",
+ "interval":10,
+ "scope": "HOST",
+ "enabled": true,
+ "source": {
+ "type": "SCRIPT",
+ "path": "DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py",
+ "parameters": [
+
+ {
+ "name": "connection.timeout",
+ "display_name": "Connection Timeout",
+ "value": 5.0,
+ "type": "NUMERIC",
+ "description": "The maximum time before this alert is considered to be CRITICAL",
+ "units": "seconds",
+ "threshold": "CRITICAL"
+ },
+ {
+ "name": "alertName",
+ "display_name": "alertName",
+ "value": "DOLPHIN_WORKER",
+ "type": "STRING",
+ "description": "alert name"
+ }
+ ]
+ }
+ }
+ ],
+ "DOLPHIN_ALERT": [
+ {
+ "name": "DOLPHIN_DOLPHIN_ALERT_CHECK",
+ "label": "check dolphin scheduler alert status",
+ "description": "",
+ "interval":10,
+ "scope": "HOST",
+ "enabled": true,
+ "source": {
+ "type": "SCRIPT",
+ "path": "DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py",
+ "parameters": [
+
+ {
+ "name": "connection.timeout",
+ "display_name": "Connection Timeout",
+ "value": 5.0,
+ "type": "NUMERIC",
+ "description": "The maximum time before this alert is considered to be CRITICAL",
+ "units": "seconds",
+ "threshold": "CRITICAL"
+ },
+ {
+ "name": "alertName",
+ "display_name": "alertName",
+ "value": "DOLPHIN_ALERT",
+ "type": "STRING",
+ "description": "alert name"
+ }
+ ]
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-alert.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-alert.xml
new file mode 100644
index 0000000..32abcc7
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-alert.xml
@@ -0,0 +1,137 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>alert.type</name>
+ <value>EMAIL</value>
+ <description>alert type is EMAIL/SMS</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.protocol</name>
+ <value>SMTP</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.server.host</name>
+ <value>xxx.xxx.com</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.server.port</name>
+ <value>25</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.sender</name>
+ <value>admin</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.user</name>
+ <value>admin</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.passwd</name>
+ <value>000000</value>
+ <description></description>
+ <property-type>PASSWORD</property-type>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>mail.smtp.starttls.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.smtp.ssl.enable</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>mail.smtp.ssl.trust</name>
+ <value>xxx.xxx.com</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>enterprise.wechat.enable</name>
+ <value>false</value>
+ <description></description>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>enterprise.wechat.corp.id</name>
+ <value>wechatId</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>enterprise.wechat.secret</name>
+ <value>secret</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>enterprise.wechat.agent.id</name>
+ <value>agentId</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>enterprise.wechat.users</name>
+ <value>wechatUsers</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-application-api.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-application-api.xml
new file mode 100644
index 0000000..766c0f4
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-application-api.xml
@@ -0,0 +1,87 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>server.port</name>
+ <value>12345</value>
+ <description>
+ server port
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ </property>
+ <property>
+ <name>server.servlet.session.timeout</name>
+ <value>7200</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ </property>
+ <property>
+ <name>server.servlet.context-path</name>
+ <value>/dolphinscheduler/</value>
+ <description>
+ </description>
+ </property>
+ <property>
+ <name>spring.servlet.multipart.max-file-size</name>
+ <value>1024</value>
+ <value-attributes>
+ <unit>MB</unit>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ </property>
+ <property>
+ <name>spring.servlet.multipart.max-request-size</name>
+ <value>1024</value>
+ <value-attributes>
+ <unit>MB</unit>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ </property>
+ <property>
+ <name>server.jetty.max-http-post-size</name>
+ <value>5000000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ </property>
+ <property>
+ <name>spring.messages.encoding</name>
+ <value>UTF-8</value>
+ <description></description>
+ </property>
+ <property>
+ <name>spring.messages.basename</name>
+ <value>i18n/messages</value>
+ <description></description>
+ </property>
+ <property>
+ <name>security.authentication.type</name>
+ <value>PASSWORD</value>
+ <description></description>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-common.xml
new file mode 100644
index 0000000..439e211
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-common.xml
@@ -0,0 +1,158 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>resource.storage.type</name>
+ <display-name>Choose Resource Upload Startup Type</display-name>
+ <description>
+ Resource upload startup type : HDFS,S3,NONE
+ </description>
+ <value>NONE</value>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>HDFS</value>
+ <label>HDFS</label>
+ </entry>
+ <entry>
+ <value>S3</value>
+ <label>S3</label>
+ </entry>
+ <entry>
+ <value>NONE</value>
+ <label>NONE</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>resource.upload.path</name>
+ <value>/dolphinscheduler</value>
+ <description>
+ resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>data.basedir.path</name>
+ <value>/tmp/dolphinscheduler</value>
+ <description>
+ user data local directory path, please make sure the directory exists and have read write permissions
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>hadoop.security.authentication.startup.state</name>
+ <value>false</value>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <description>whether kerberos starts</description>
+ </property>
+ <property>
+ <name>java.security.krb5.conf.path</name>
+ <value>/opt/krb5.conf</value>
+ <description>
+ java.security.krb5.conf path
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>login.user.keytab.username</name>
+ <value>hdfs-mycluster@ESZ.COM</value>
+ <description>
+ LoginUserFromKeytab user
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>login.user.keytab.path</name>
+ <value>/opt/hdfs.headless.keytab</value>
+ <description>
+ LoginUserFromKeytab path
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>resource.view.suffixs</name>
+ <value>txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties</value>
+ <description></description>
+ </property>
+ <property>
+ <name>hdfs.root.user</name>
+ <value>hdfs</value>
+ <description>
+ Users who have permission to create directories under the HDFS root path
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://mycluster:8020</value>
+ <description>
+ HA or single namenode,
+ If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory，
+ support s3，for example : s3a://dolphinscheduler
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>http://host:9010</value>
+ <description>
+ s3 need，s3 endpoint
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>A3DXS30FO22544RE</value>
+ <description>
+ s3 need，s3 access key
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>OloCLq3n+8+sdPHUhJ21XrSxTC+JK</value>
+ <description>
+ s3 need，s3 secret key
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kerberos.expire.time</name>
+ <value>7</value>
+ <description></description>
+ </property>
+
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-datasource.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-datasource.xml
new file mode 100644
index 0000000..02d8de0
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-datasource.xml
@@ -0,0 +1,206 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>spring.datasource.initialSize</name>
+ <value>5</value>
+ <description>
+ Init connection number
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.minIdle</name>
+ <value>5</value>
+ <description>
+ Min connection number
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.maxActive</name>
+ <value>50</value>
+ <description>
+ Max connection number
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.maxWait</name>
+ <value>60000</value>
+ <description>
+ Max wait time for get a connection in milliseconds.
+ If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases.
+ If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true.
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.timeBetweenEvictionRunsMillis</name>
+ <value>60000</value>
+ <description>
+ Milliseconds for check to close free connections
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.timeBetweenConnectErrorMillis</name>
+ <value>60000</value>
+ <description>
+ The Destroy thread detects the connection interval and closes the physical connection in milliseconds
+ if the connection idle time is greater than or equal to minEvictableIdleTimeMillis.
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.minEvictableIdleTimeMillis</name>
+ <value>300000</value>
+ <description>
+ The longest time a connection remains idle without being evicted, in milliseconds
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.validationQuery</name>
+ <value>SELECT 1</value>
+ <description>
+ The SQL used to check whether the connection is valid requires a query statement.
+ If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.validationQueryTimeout</name>
+ <value>3</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ Check whether the connection is valid for timeout, in seconds
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.testWhileIdle</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ When applying for a connection,
+ if it is detected that the connection is idle longer than time Between Eviction Runs Millis,
+ validation Query is performed to check whether the connection is valid
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.testOnBorrow</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ Execute validation to check if the connection is valid when applying for a connection
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.testOnReturn</name>
+ <value>false</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ Execute validation to check if the connection is valid when the connection is returned
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.defaultAutoCommit</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.keepAlive</name>
+ <value>false</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>spring.datasource.poolPreparedStatements</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description>
+ Open PSCache, specify count PSCache for every connection
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.maxPoolPreparedStatementPerConnectionSize</name>
+ <value>20</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.spring.datasource.filters</name>
+ <value>stat,wall,log4j</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spring.datasource.connectionProperties</name>
+ <value>druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000</value>
+ <description></description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-env.xml
new file mode 100644
index 0000000..8e14716
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-env.xml
@@ -0,0 +1,123 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>dolphin.database.type</name>
+ <value>mysql</value>
+ <description>Dolphin Scheduler DataBase Type Which Is Select</description>
+ <display-name>Dolphin Database Type</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>mysql</value>
+ <label>Mysql</label>
+ </entry>
+ <entry>
+ <value>postgresql</value>
+ <label>Postgresql</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>dolphin.database.host</name>
+ <value></value>
+ <display-name>Dolphin Database Host</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>dolphin.database.port</name>
+ <value></value>
+ <display-name>Dolphin Database Port</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>dolphin.database.username</name>
+ <value></value>
+ <display-name>Dolphin Database Username</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>dolphin.database.password</name>
+ <value></value>
+ <display-name>Dolphin Database Password</display-name>
+ <property-type>PASSWORD</property-type>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>dolphin.user</name>
+ <value></value>
+ <description>Which user to install and admin dolphin scheduler</description>
+ <display-name>Deploy User</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>dolphin.group</name>
+ <value></value>
+ <description>Which user to install and admin dolphin scheduler</description>
+ <display-name>Deploy Group</display-name>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>dolphinscheduler-env-content</name>
+ <display-name>Dolphinscheduler Env template</display-name>
+ <description>This is the jinja template for dolphinscheduler.env.sh file</description>
+ <value>#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+export HADOOP_HOME=/opt/soft/hadoop
+export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
+export SPARK_HOME1=/opt/soft/spark1
+export SPARK_HOME2=/opt/soft/spark2
+export PYTHON_HOME=/opt/soft/python
+export JAVA_HOME=/opt/soft/java
+export HIVE_HOME=/opt/soft/hive
+export FLINK_HOME=/opt/soft/flink</value>
+ <value-attributes>
+ <type>content</type>
+ <empty-value-valid>false</empty-value-valid>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-master.xml
new file mode 100644
index 0000000..c8eec04
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-master.xml
@@ -0,0 +1,88 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>master.exec.threads</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>master execute thread num</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>master.exec.task.num</name>
+ <value>20</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>master execute task number in parallel</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>master.heartbeat.interval</name>
+ <value>10</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>master heartbeat interval</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>master.task.commit.retryTimes</name>
+ <value>5</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>master commit task retry times</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>master.task.commit.interval</name>
+ <value>1000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>master commit task interval</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>master.max.cpuload.avg</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>only less than cpu avg load, master server can work. default value : the number of cpu cores * 2</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>master.reserved.memory</name>
+ <value>0.3</value>
+ <description>only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>master.listen.port</name>
+ <value>5678</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>master listen port</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-quartz.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-quartz.xml
new file mode 100644
index 0000000..7a0c68b
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-quartz.xml
@@ -0,0 +1,126 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>org.quartz.scheduler.instanceName</name>
+ <value>DolphinScheduler</value>
+ <description></description>
+ </property>
+ <property>
+ <!-- 列举枚举值 -->
+ <name>org.quartz.scheduler.instanceId</name>
+ <value>AUTO</value>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.scheduler.makeSchedulerThreadDaemon</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.useProperties</name>
+ <value>false</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.threadPool.class</name>
+ <value>org.quartz.simpl.SimpleThreadPool</value>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.threadPool.makeThreadsDaemons</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.threadPool.threadCount</name>
+ <value>25</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.threadPool.threadPriority</name>
+ <value>5</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.class</name>
+ <value>org.quartz.impl.jdbcjobstore.JobStoreTX</value>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.tablePrefix</name>
+ <value>QRTZ_</value>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.isClustered</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.misfireThreshold</name>
+ <value>60000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.clusterCheckinInterval</name>
+ <value>5000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.acquireTriggersWithinLock</name>
+ <value>true</value>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.jobStore.dataSource</name>
+ <value>myDs</value>
+ <description></description>
+ </property>
+ <property>
+ <name>org.quartz.dataSource.myDs.connectionProvider.class</name>
+ <value>org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider</value>
+ <description></description>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-worker.xml
new file mode 100644
index 0000000..1ae7a1a
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-worker.xml
@@ -0,0 +1,67 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>worker.exec.threads</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>worker execute thread num</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>worker.heartbeat.interval</name>
+ <value>10</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>worker heartbeat interval</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>worker.max.cpuload.avg</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>worker.reserved.memory</name>
+ <value>0.3</value>
+ <description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>worker.listen.port</name>
+ <value>1234</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>worker listen port</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>worker.groups</name>
+ <value>default</value>
+ <description>default worker group</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-zookeeper.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-zookeeper.xml
new file mode 100644
index 0000000..e89962d
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/configuration/dolphin-zookeeper.xml
@@ -0,0 +1,76 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>zookeeper.dolphinscheduler.root</name>
+ <value>/dolphinscheduler</value>
+ <description>
+ dolphinscheduler root directory
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value>300</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>zookeeper.connection.timeout</name>
+ <value>300</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>zookeeper.retry.base.sleep</name>
+ <value>100</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>zookeeper.retry.max.sleep</name>
+ <value>30000</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>zookeeper.retry.maxtime</name>
+ <value>5</value>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <description>
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/1.3.4/metainfo.xml
new file mode 100644
index 0000000..c25ded5
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/metainfo.xml
@@ -0,0 +1,137 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>DOLPHIN</name>
+ <displayName>Dolphin Scheduler</displayName>
+ <comment>分布式易扩展的可视化DAG工作流任务调度系统</comment>
+ <version>1.3.4</version>
+ <components>
+ <component>
+ <name>DOLPHIN_MASTER</name>
+ <displayName>DS Master</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/dolphin_master_service.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DOLPHIN_LOGGER</name>
+ <displayName>DS Logger</displayName>
+ <category>SLAVE</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/dolphin_logger_service.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DOLPHIN_WORKER</name>
+ <displayName>DS Worker</displayName>
+ <category>SLAVE</category>
+ <cardinality>1+</cardinality>
+ <dependencies>
+ <dependency>
+ <name>DOLPHIN/DOLPHIN_LOGGER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/dolphin_worker_service.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DOLPHIN_ALERT</name>
+ <displayName>DS Alert</displayName>
+ <category>SLAVE</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/dolphin_alert_service.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DOLPHIN_API</name>
+ <displayName>DS_Api</displayName>
+ <category>SLAVE</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/dolphin_api_service.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <requiredServices>
+ <service>ZOOKEEPER</service>
+ </requiredServices>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>apache-dolphinscheduler-incubating*</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configuration-dependencies>
+ <config-type>dolphin-alert</config-type>
+ <config-type>dolphin-app-api</config-type>
+ <config-type>dolphin-app-dao</config-type>
+ <config-type>dolphin-common</config-type>
+ <config-type>dolphin-env</config-type>
+ <config-type>dolphin-quartz</config-type>
+ </configuration-dependencies>
+
+ <themes>
+ <theme>
+ <fileName>theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ <quickLinksConfigurations-dir>quicklinks</quickLinksConfigurations-dir>
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py
new file mode 100644
index 0000000..87cc7b4
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/alerts/alert_dolphin_scheduler_status.py
@@ -0,0 +1,124 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import socket
+import urllib2
+import os
+import logging
+import ambari_simplejson as json
+from resource_management.libraries.script.script import Script
+import sys
+reload(sys)
+sys.setdefaultencoding('utf-8')
+
+logger = logging.getLogger('ambari_alerts')
+
+config = Script.get_config()
+
+
+def get_tokens():
+ """
+ Returns a tuple of tokens in the format {{site/property}} that will be used
+ to build the dictionary passed into execute
+
+ :rtype tuple
+ """
+
+def get_info(url, connection_timeout):
+ response = None
+
+ try:
+ response = urllib2.urlopen(url, timeout=connection_timeout)
+ json_data = response.read()
+ return json_data
+ finally:
+ if response is not None:
+ try:
+ response.close()
+ except:
+ pass
+
+
+def execute(configurations={}, parameters={}, host_name=None):
+ """
+ Returns a tuple containing the result code and a pre-formatted result label
+
+ Keyword arguments:
+ configurations : a mapping of configuration key to value
+ parameters : a mapping of script parameter key to value
+ host_name : the name of this host where the alert is running
+
+ :type configurations dict
+ :type parameters dict
+ :type host_name str
+ """
+
+ alert_name = parameters['alertName']
+
+ dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler"
+
+ pid = "0"
+
+
+ from resource_management.core import sudo
+
+ is_running = True
+ pid_file_path = ""
+ if alert_name == 'DOLPHIN_MASTER':
+ pid_file_path = dolphin_pidfile_dir + "/master-server.pid"
+ elif alert_name == 'DOLPHIN_WORKER':
+ pid_file_path = dolphin_pidfile_dir + "/worker-server.pid"
+ elif alert_name == 'DOLPHIN_ALERT':
+ pid_file_path = dolphin_pidfile_dir + "/alert-server.pid"
+ elif alert_name == 'DOLPHIN_LOGGER':
+ pid_file_path = dolphin_pidfile_dir + "/logger-server.pid"
+ elif alert_name == 'DOLPHIN_API':
+ pid_file_path = dolphin_pidfile_dir + "/api-server.pid"
+
+ if not pid_file_path or not os.path.isfile(pid_file_path):
+ is_running = False
+
+ try:
+ pid = int(sudo.read_file(pid_file_path))
+ except:
+ is_running = False
+
+ try:
+ # Kill will not actually kill the process
+ # From the doc:
+ # If sig is 0, then no signal is sent, but error checking is still
+ # performed; this can be used to check for the existence of a
+ # process ID or process group ID.
+ sudo.kill(pid, 0)
+ except OSError:
+ is_running = False
+
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ if not is_running:
+ result_code = "CRITICAL"
+ else:
+ result_code = "OK"
+
+ label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code)
+
+ return ((result_code, [label]))
+
+if __name__ == "__main__":
+ pass
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_alert_service.py
new file mode 100644
index 0000000..e78c38d
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_alert_service.py
@@ -0,0 +1,62 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import time
+from resource_management import *
+
+from dolphin_env import dolphin_env
+
+
+class DolphinAlertService(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
+
+ def configure(self, env):
+ import params
+ params.pika_slave = True
+ env.set_params(params)
+
+ dolphin_env()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1")
+
+ start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server")
+ Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server")
+ Execute(stop_cmd, user=params.dolphin_user)
+ time.sleep(5)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.dolphin_run_dir + "alert-server.pid")
+
+
+if __name__ == "__main__":
+ DolphinAlertService().execute()
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_api_service.py
new file mode 100644
index 0000000..5a28924
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_api_service.py
@@ -0,0 +1,71 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import time
+from resource_management import *
+
+from dolphin_env import dolphin_env
+
+
+class DolphinApiService(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
+
+ def configure(self, env):
+ import params
+ params.pika_slave = True
+ env.set_params(params)
+
+ dolphin_env()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ #init
+ init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh")
+ Execute(init_cmd, user=params.dolphin_user)
+
+ #upgrade
+ upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh")
+ Execute(upgrade_cmd, user=params.dolphin_user)
+
+ no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1")
+
+ start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server")
+ Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server")
+ Execute(stop_cmd, user=params.dolphin_user)
+ time.sleep(5)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.dolphin_run_dir + "api-server.pid")
+
+
+if __name__ == "__main__":
+ DolphinApiService().execute()
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_env.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_env.py
new file mode 100644
index 0000000..1661d76
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_env.py
@@ -0,0 +1,123 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+
+def dolphin_env():
+ import params
+
+ Directory(params.dolphin_pidfile_dir,
+ mode=0777,
+ owner=params.dolphin_user,
+ group=params.dolphin_group,
+ create_parents=True
+ )
+ Directory(params.dolphin_log_dir,
+ mode=0777,
+ owner=params.dolphin_user,
+ group=params.dolphin_group,
+ create_parents=True
+ )
+ Directory(params.dolphin_conf_dir,
+ mode=0777,
+ owner=params.dolphin_user,
+ group=params.dolphin_group,
+ create_parents=True
+ )
+
+ Directory(params.dolphin_common_map['data.basedir.path'],
+ mode=0777,
+ owner=params.dolphin_user,
+ group=params.dolphin_group,
+ create_parents=True
+ )
+
+
+ File(format(params.dolphin_env_path),
+ mode=0777,
+ content=InlineTemplate(params.dolphin_env_content),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+
+ File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"),
+ mode=0755,
+ content=Template("dolphin-daemon.sh.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+ File(format(params.dolphin_conf_dir + "/master.properties"),
+ mode=0755,
+ content=Template("master.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+ File(format(params.dolphin_conf_dir + "/worker.properties"),
+ mode=0755,
+ content=Template("worker.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+
+ File(format(params.dolphin_conf_dir + "/alert.properties"),
+ mode=0755,
+ content=Template("alert.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+ File(format(params.dolphin_conf_dir + "/datasource.properties"),
+ mode=0755,
+ content=Template("datasource.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+ File(format(params.dolphin_conf_dir + "/application-api.properties"),
+ mode=0755,
+ content=Template("application-api.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+ File(format(params.dolphin_conf_dir + "/common.properties"),
+ mode=0755,
+ content=Template("common.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+ File(format(params.dolphin_conf_dir + "/quartz.properties"),
+ mode=0755,
+ content=Template("quartz.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
+
+ File(format(params.dolphin_conf_dir + "/zookeeper.properties"),
+ mode=0755,
+ content=Template("zookeeper.properties.j2"),
+ owner=params.dolphin_user,
+ group=params.dolphin_group
+ )
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_logger_service.py
new file mode 100644
index 0000000..fb47e13
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_logger_service.py
@@ -0,0 +1,61 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import time
+from resource_management import *
+
+from dolphin_env import dolphin_env
+
+
+class DolphinLoggerService(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
+ def configure(self, env):
+ import params
+ params.pika_slave = True
+ env.set_params(params)
+
+ dolphin_env()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1")
+
+ start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server")
+ Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server")
+ Execute(stop_cmd, user=params.dolphin_user)
+ time.sleep(5)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.dolphin_run_dir + "logger-server.pid")
+
+
+if __name__ == "__main__":
+ DolphinLoggerService().execute()
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_master_service.py
new file mode 100644
index 0000000..8d64935
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_master_service.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import time
+from resource_management import *
+
+from dolphin_env import dolphin_env
+
+
+class DolphinMasterService(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
+
+ def configure(self, env):
+ import params
+ params.pika_slave = True
+ env.set_params(params)
+
+ dolphin_env()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1")
+ start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server")
+ Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server")
+ Execute(stop_cmd, user=params.dolphin_user)
+ time.sleep(5)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.dolphin_run_dir + "master-server.pid")
+
+
+if __name__ == "__main__":
+ DolphinMasterService().execute()
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_worker_service.py
new file mode 100644
index 0000000..1f542c0
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/dolphin_worker_service.py
@@ -0,0 +1,61 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import time
+from resource_management import *
+
+from dolphin_env import dolphin_env
+
+
+class DolphinWorkerService(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
+
+ def configure(self, env):
+ import params
+ params.pika_slave = True
+ env.set_params(params)
+
+ dolphin_env()
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1")
+ start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server")
+ Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test)
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+ stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server")
+ Execute(stop_cmd, user=params.dolphin_user)
+ time.sleep(5)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.dolphin_run_dir + "worker-server.pid")
+
+
+if __name__ == "__main__":
+ DolphinWorkerService().execute()
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/params.py
new file mode 100644
index 0000000..3ca46f2
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/params.py
@@ -0,0 +1,155 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+import sys
+from resource_management import *
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import default
+
+Logger.initialize_logger()
+reload(sys)
+sys.setdefaultencoding('utf-8')
+
+# server configurations
+config = Script.get_config()
+
+# conf_dir = "/etc/"
+dolphin_home = "/opt/soft/dolphinscheduler"
+dolphin_conf_dir = dolphin_home + "/conf"
+dolphin_log_dir = dolphin_home + "/logs"
+dolphin_bin_dir = dolphin_home + "/bin"
+dolphin_lib_jars = dolphin_home + "/lib/*"
+dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler"
+
+rmHosts = default("/clusterHostInfo/rm_host", [])
+
+# dolphin-env
+dolphin_env_map = {}
+dolphin_env_map.update(config['configurations']['dolphin-env'])
+
+# which user to install and admin dolphin scheduler
+dolphin_user = dolphin_env_map['dolphin.user']
+dolphin_group = dolphin_env_map['dolphin.group']
+
+# .dolphinscheduler_env.sh
+dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh'
+dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content']
+
+# database config
+dolphin_database_config = {}
+dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type']
+dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username']
+dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password']
+if 'mysql' == dolphin_database_config['dolphin_database_type']:
+ dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver'
+ dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate'
+ dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \
+ + ':' + dolphin_env_map['dolphin.database.port'] \
+ + '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8'
+else:
+ dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver'
+ dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate'
+ dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \
+ + ':' + dolphin_env_map['dolphin.database.port'] \
+ + '/dolphinscheduler'
+
+
+
+
+
+# application-alert.properties
+dolphin_alert_map = {}
+wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
+wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
+wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}'
+wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}'
+
+dolphin_alert_config_map = config['configurations']['dolphin-alert']
+
+if dolphin_alert_config_map['enterprise.wechat.enable']:
+ dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url
+ dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url
+ dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg
+ dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg
+
+dolphin_alert_map.update(dolphin_alert_config_map)
+
+
+
+# application-api.properties
+dolphin_app_api_map = {}
+dolphin_app_api_map.update(config['configurations']['dolphin-application-api'])
+
+
+# common.properties
+dolphin_common_map = {}
+
+if 'yarn-site' in config['configurations'] and \
+ 'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']:
+ yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address']
+ yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s'
+ dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address
+
+rmHosts = default("/clusterHostInfo/rm_host", [])
+if len(rmHosts) > 1:
+ dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts)
+else:
+ dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ''
+
+dolphin_common_map_tmp = config['configurations']['dolphin-common']
+data_basedir_path = dolphin_common_map_tmp['data.basedir.path']
+dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path
+dolphin_common_map.update(config['configurations']['dolphin-common'])
+
+# datasource.properties
+dolphin_datasource_map = {}
+dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource'
+dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver']
+dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url']
+dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username']
+dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password']
+dolphin_datasource_map.update(config['configurations']['dolphin-datasource'])
+
+# master.properties
+dolphin_master_map = config['configurations']['dolphin-master']
+
+# quartz.properties
+dolphin_quartz_map = {}
+dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass']
+dolphin_quartz_map.update(config['configurations']['dolphin-quartz'])
+
+# worker.properties
+dolphin_worker_map = config['configurations']['dolphin-worker']
+
+# zookeeper.properties
+dolphin_zookeeper_map={}
+zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", [])
+if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']:
+ clientPort = config['configurations']['zoo.cfg']['clientPort']
+ zookeeperPort = ":" + clientPort + ","
+ dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort
+dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper'])
+if 'spring.servlet.multipart.max-file-size' in dolphin_app_api_map:
+ file_size = dolphin_app_api_map['spring.servlet.multipart.max-file-size']
+ dolphin_app_api_map['spring.servlet.multipart.max-file-size'] = file_size + "MB"
+if 'spring.servlet.multipart.max-request-size' in dolphin_app_api_map:
+ request_size = dolphin_app_api_map['spring.servlet.multipart.max-request-size']
+ dolphin_app_api_map['spring.servlet.multipart.max-request-size'] = request_size + "MB"
+
+
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/service_check.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/service_check.py
new file mode 100644
index 0000000..0e12f69
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/service_check.py
@@ -0,0 +1,31 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions import get_unique_id_and_date
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ #env.set_params(params)
+
+ # Execute(format("which pika_server"))
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/status_params.py b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/status_params.py
new file mode 100644
index 0000000..24b2c8b
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/scripts/status_params.py
@@ -0,0 +1,23 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+dolphin_run_dir = "/opt/soft/run/dolphinscheduler/"
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/alert.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/alert.properties.j2
index 362a4e8..73840b8 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/alert.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_alert_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/application-api.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/application-api.properties.j2
index 362a4e8..7011800 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/application-api.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_app_api_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/common.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/common.properties.j2
index 362a4e8..2220c4e 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/common.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_common_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/datasource.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/datasource.properties.j2
index 362a4e8..40aed83 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/datasource.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_datasource_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/dolphin-daemon.sh.j2 b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/dolphin-daemon.sh.j2
new file mode 100644
index 0000000..0802b74
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/dolphin-daemon.sh.j2
@@ -0,0 +1,116 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+usage="Usage: dolphinscheduler-daemon.sh (start|stop) <command> "
+
+# if no args specified, show usage
+if [ $# -le 1 ]; then
+ echo $usage
+ exit 1
+fi
+
+startStop=$1
+shift
+command=$1
+shift
+
+echo "Begin $startStop $command......"
+
+BIN_DIR=`dirname $0`
+BIN_DIR=`cd "$BIN_DIR"; pwd`
+DOLPHINSCHEDULER_HOME=$BIN_DIR/..
+
+export HOSTNAME=`hostname`
+
+DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}}
+
+DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+STOP_TIMEOUT=5
+
+log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out
+pid={{dolphin_pidfile_dir}}/$command.pid
+
+cd $DOLPHINSCHEDULER_HOME
+
+if [ "$command" = "api-server" ]; then
+ LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api"
+ CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
+elif [ "$command" = "master-server" ]; then
+ LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false"
+ CLASS=org.apache.dolphinscheduler.server.master.MasterServer
+elif [ "$command" = "worker-server" ]; then
+ LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false"
+ CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
+elif [ "$command" = "alert-server" ]; then
+ LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml"
+ CLASS=org.apache.dolphinscheduler.alert.AlertServer
+elif [ "$command" = "logger-server" ]; then
+ CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
+else
+ echo "Error: No command named \`$command' was found."
+ exit 1
+fi
+
+case $startStop in
+ (start)
+
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid` > /dev/null 2>&1; then
+ echo $command running as process `cat $pid`. Stop it first.
+ exit 1
+ fi
+ fi
+
+ echo starting $command, logging to $log
+
+ exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS"
+
+ echo "nohup java $exec_command > $log 2>&1 < /dev/null &"
+ nohup java $exec_command > $log 2>&1 < /dev/null &
+ echo $! > $pid
+ ;;
+
+ (stop)
+
+ if [ -f $pid ]; then
+ TARGET_PID=`cat $pid`
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ echo stopping $command
+ kill $TARGET_PID
+ sleep $STOP_TIMEOUT
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9"
+ kill -9 $TARGET_PID
+ fi
+ else
+ echo no $command to stop
+ fi
+ rm -f $pid
+ else
+ echo no $command to stop
+ fi
+ ;;
+
+ (*)
+ echo $usage
+ exit 1
+ ;;
+
+esac
+
+echo "End $startStop $command."
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/master.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/master.properties.j2
index 362a4e8..d9b85e1 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/master.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_master_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/quartz.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/quartz.properties.j2
index 362a4e8..e027a26 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/quartz.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_quartz_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/worker.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/worker.properties.j2
index 362a4e8..a008b74 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/worker.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_worker_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/zookeeper.properties.j2
similarity index 64%
copy from docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
copy to ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/zookeeper.properties.j2
index 362a4e8..9eb14ea 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/package/templates/zookeeper.properties.j2
@@ -15,15 +15,6 @@
# limitations under the License.
#
-# zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181
-zookeeper.quorum=${ZOOKEEPER_QUORUM}
-
-# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
-
-# dolphinscheduler failover directory
-#zookeeper.session.timeout=60000
-#zookeeper.connection.timeout=30000
-#zookeeper.retry.base.sleep=100
-#zookeeper.retry.max.sleep=30000
-#zookeeper.retry.maxtime=10
\ No newline at end of file
+{% for key, value in dolphin_zookeeper_map.iteritems() -%}
+ {{key}}={{value}}
+{% endfor %}
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/quicklinks/quicklinks.json b/ambari_plugin/common-services/DOLPHIN/1.3.4/quicklinks/quicklinks.json
new file mode 100644
index 0000000..8753004
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/quicklinks/quicklinks.json
@@ -0,0 +1,26 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol":
+ {
+ "type":"http"
+ },
+
+ "links": [
+ {
+ "name": "dolphin-application-ui",
+ "label": "DolphinApplication UI",
+ "requires_user_name": "false",
+ "component_name": "DOLPHIN_API",
+ "url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html",
+ "port":{
+ "http_property": "server.port",
+ "http_default_port": "12345",
+ "regex": "^(\\d+)$",
+ "site": "dolphin-application-api"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.4/themes/theme.json b/ambari_plugin/common-services/DOLPHIN/1.3.4/themes/theme.json
new file mode 100644
index 0000000..953e232
--- /dev/null
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.4/themes/theme.json
@@ -0,0 +1,661 @@
+{
+ "name": "default",
+ "description": "Default theme for Dolphin Scheduler service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "settings",
+ "display-name": "Settings",
+ "layout": {
+ "tab-rows": "3",
+ "tab-columns": "3",
+ "sections": [
+ {
+ "name": "dolphin-env-config",
+ "display-name": "Dolphin Env Config",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "2",
+ "section-rows": "1",
+ "section-columns": "2",
+ "subsections": [
+ {
+ "name": "env-row1-col1",
+ "display-name": "Deploy User Info",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "env-row1-col2",
+ "display-name": "System Env Optimization",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "dolphin-database-config",
+ "display-name": "Database Config",
+ "row-index": "1",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "2",
+ "section-rows": "1",
+ "section-columns": "3",
+ "subsections": [
+ {
+ "name": "database-row1-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "database-row1-col2",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "database-row1-col3",
+ "row-index": "0",
+ "column-index": "2",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "dynamic-config",
+ "row-index": "2",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "2",
+ "section-rows": "1",
+ "section-columns": "3",
+ "subsections": [
+ {
+ "name": "dynamic-row1-col1",
+ "display-name": "Resource FS Config",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "dynamic-row1-col2",
+ "display-name": "Kerberos Info",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "dynamic-row1-col3",
+ "display-name": "Wechat Info",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "dolphin-env/dolphin.database.type",
+ "subsection-name": "database-row1-col1"
+ },
+ {
+ "config": "dolphin-env/dolphin.database.host",
+ "subsection-name": "database-row1-col2"
+ },
+ {
+ "config": "dolphin-env/dolphin.database.port",
+ "subsection-name": "database-row1-col2"
+ },
+ {
+ "config": "dolphin-env/dolphin.database.username",
+ "subsection-name": "database-row1-col3"
+ },
+ {
+ "config": "dolphin-env/dolphin.database.password",
+ "subsection-name": "database-row1-col3"
+ },
+ {
+ "config": "dolphin-env/dolphin.user",
+ "subsection-name": "env-row1-col1"
+ },
+ {
+ "config": "dolphin-env/dolphin.group",
+ "subsection-name": "env-row1-col1"
+ },
+ {
+ "config": "dolphin-env/dolphinscheduler-env-content",
+ "subsection-name": "env-row1-col2"
+ },
+ {
+ "config": "dolphin-common/resource.storage.type",
+ "subsection-name": "dynamic-row1-col1"
+ },
+ {
+ "config": "dolphin-common/resource.upload.path",
+ "subsection-name": "dynamic-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/resource.storage.type"
+ ],
+ "if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/hdfs.root.user",
+ "subsection-name": "dynamic-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/resource.storage.type"
+ ],
+ "if": "${dolphin-common/resource.storage.type} === HDFS",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/data.store2hdfs.basepath",
+ "subsection-name": "dynamic-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/resource.storage.type"
+ ],
+ "if": "${dolphin-common/resource.storage.type} === HDFS",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/fs.defaultFS",
+ "subsection-name": "dynamic-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/resource.storage.type"
+ ],
+ "if": "${dolphin-common/resource.storage.type} === HDFS",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/fs.s3a.endpoint",
+ "subsection-name": "dynamic-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/resource.storage.type"
+ ],
+ "if": "${dolphin-common/resource.storage.type} === S3",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/fs.s3a.access.key",
+ "subsection-name": "dynamic-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/resource.storage.type"
+ ],
+ "if": "${dolphin-common/resource.storage.type} === S3",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/fs.s3a.secret.key",
+ "subsection-name": "dynamic-row1-col1",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/resource.storage.type"
+ ],
+ "if": "${dolphin-common/resource.storage.type} === S3",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/hadoop.security.authentication.startup.state",
+ "subsection-name": "dynamic-row1-col2"
+ },
+ {
+ "config": "dolphin-common/java.security.krb5.conf.path",
+ "subsection-name": "dynamic-row1-col2",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/hadoop.security.authentication.startup.state"
+ ],
+ "if": "${dolphin-common/hadoop.security.authentication.startup.state}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/login.user.keytab.username",
+ "subsection-name": "dynamic-row1-col2",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/hadoop.security.authentication.startup.state"
+ ],
+ "if": "${dolphin-common/hadoop.security.authentication.startup.state}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/login.user.keytab.path",
+ "subsection-name": "dynamic-row1-col2",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/hadoop.security.authentication.startup.state"
+ ],
+ "if": "${dolphin-common/hadoop.security.authentication.startup.state}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-common/kerberos.expire.time",
+ "subsection-name": "dynamic-row1-col2",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-common/hadoop.security.authentication.startup.state"
+ ],
+ "if": "${dolphin-common/hadoop.security.authentication.startup.state}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.enable",
+ "subsection-name": "dynamic-row1-col3"
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.corp.id",
+ "subsection-name": "dynamic-row1-col3",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-alert/enterprise.wechat.enable"
+ ],
+ "if": "${dolphin-alert/enterprise.wechat.enable}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.secret",
+ "subsection-name": "dynamic-row1-col3",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-alert/enterprise.wechat.enable"
+ ],
+ "if": "${dolphin-alert/enterprise.wechat.enable}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.agent.id",
+ "subsection-name": "dynamic-row1-col3",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-alert/enterprise.wechat.enable"
+ ],
+ "if": "${dolphin-alert/enterprise.wechat.enable}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.users",
+ "subsection-name": "dynamic-row1-col3",
+ "depends-on": [
+ {
+ "configs":[
+ "dolphin-alert/enterprise.wechat.enable"
+ ],
+ "if": "${dolphin-alert/enterprise.wechat.enable}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "dolphin-env/dolphin.database.type",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "dolphin-env/dolphin.database.host",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-env/dolphin.database.port",
+ "widget": {
+ "type": "text-field",
+ "units": [
+ {
+ "unit-name": "int"
+ }
+ ]
+ }
+ },
+ {
+ "config": "dolphin-env/dolphin.database.username",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-env/dolphin.database.password",
+ "widget": {
+ "type": "password"
+ }
+ },
+ {
+ "config": "dolphin-env/dolphin.user",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-env/dolphin.group",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-env/dolphinscheduler-env-content",
+ "widget": {
+ "type": "text-area"
+ }
+ },
+ {
+ "config": "dolphin-common/resource.storage.type",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "dolphin-common/resource.upload.path",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/hdfs.root.user",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/data.store2hdfs.basepath",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/fs.defaultFS",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/fs.s3a.endpoint",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/fs.s3a.access.key",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/fs.s3a.secret.key",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/hadoop.security.authentication.startup.state",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "dolphin-common/java.security.krb5.conf.path",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/login.user.keytab.username",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/login.user.keytab.path",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-common/kerberos.expire.time",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.enable",
+ "widget": {
+ "type": "toggle"
+ }
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.corp.id",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.secret",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.agent.id",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "dolphin-alert/enterprise.wechat.users",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+}
diff --git a/ambari_plugin/statcks/DOLPHIN/metainfo.xml b/ambari_plugin/statcks/DOLPHIN/metainfo.xml
index 386a537..2854c49 100755
--- a/ambari_plugin/statcks/DOLPHIN/metainfo.xml
+++ b/ambari_plugin/statcks/DOLPHIN/metainfo.xml
@@ -20,7 +20,7 @@
<services>
<service>
<name>DOLPHIN</name>
- <extends>common-services/DOLPHIN/1.3.3</extends>
+ <extends>common-services/DOLPHIN/1.3.4</extends>
</service>
</services>
</metainfo>
\ No newline at end of file
diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile
index ebc2d76..a870dd3 100644
--- a/docker/build/Dockerfile
+++ b/docker/build/Dockerfile
@@ -29,8 +29,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apk update && \
apk --update add --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \
apk add --update procps && \
- openrc boot && \
- pip install kazoo
+ openrc boot
#2. install jdk
RUN apk add openjdk8
diff --git a/docker/build/README.md b/docker/build/README.md
index 5725ec5..2607c58 100644
--- a/docker/build/README.md
+++ b/docker/build/README.md
@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
+-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@@ -25,14 +25,14 @@ The default postgres user `root`, postgres password `root` and database `dolphin
The default zookeeper is created in the `startup.sh`.
-#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`**
+#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`**
You can specify **existing postgres service**. Example:
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -56,8 +56,8 @@ You can start a standalone dolphinscheduler server.
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server
```
@@ -66,8 +66,8 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server
```
@@ -75,8 +75,8 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
```
@@ -85,8 +85,8 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server
```
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
-**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
+**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
## How to build a docker image
@@ -124,33 +124,51 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script
The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image.
-**`POSTGRESQL_HOST`**
+**`DATABASE_TYPE`**
-This environment variable sets the host for PostgreSQL. The default value is `127.0.0.1`.
+This environment variable sets the type for database. The default value is `postgresql`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_PORT`**
+**`DATABASE_DRIVER`**
-This environment variable sets the port for PostgreSQL. The default value is `5432`.
+This environment variable sets the type for database. The default value is `org.postgresql.Driver`.
-**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_HOST`**
+
+This environment variable sets the host for database. The default value is `127.0.0.1`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_PORT`**
+
+This environment variable sets the port for database. The default value is `5432`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_USERNAME`**
+
+This environment variable sets the username for database. The default value is `root`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_USERNAME`**
+**`DATABASE_PASSWORD`**
-This environment variable sets the username for PostgreSQL. The default value is `root`.
+This environment variable sets the password for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_PASSWORD`**
+**`DATABASE_DATABASE`**
-This environment variable sets the password for PostgreSQL. The default value is `root`.
+This environment variable sets the database for database. The default value is `dolphinscheduler`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_DATABASE`**
+**`DATABASE_PARAMS`**
-This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`.
+This environment variable sets the database for database. The default value is `characterEncoding=utf8`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md
index 0605bef..c2affc0 100644
--- a/docker/build/README_zh_CN.md
+++ b/docker/build/README_zh_CN.md
@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
+-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@@ -25,14 +25,14 @@ dolphinscheduler all
同时，默认的`Zookeeper`也会在`startup.sh`脚本中被创建。
-#### 或者通过环境变量 **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
+#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
你可以指定一个已经存在的 **`Postgres`** 服务. 如下:
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -42,7 +42,7 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -56,8 +56,8 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server
```
@@ -66,8 +66,8 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server
```
@@ -75,8 +75,8 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
```
@@ -85,8 +85,8 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server
```
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
-**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。
+**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。
## 如何构建一个docker镜像
@@ -124,33 +124,51 @@ c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。
-**`POSTGRESQL_HOST`**
+**`DATABASE_TYPE`**
-配置`PostgreSQL`的`HOST`， 默认值 `127.0.0.1`。
+配置`database`的`TYPE`， 默认值 `postgresql`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_PORT`**
+**`DATABASE_DRIVER`**
-配置`PostgreSQL`的`PORT`， 默认值 `5432`。
+配置`database`的`DRIVER`， 默认值 `org.postgresql.Driver`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_USERNAME`**
+**`DATABASE_HOST`**
-配置`PostgreSQL`的`USERNAME`， 默认值 `root`。
+配置`database`的`HOST`， 默认值 `127.0.0.1`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_PASSWORD`**
+**`DATABASE_PORT`**
-配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。
+配置`database`的`PORT`， 默认值 `5432`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_DATABASE`**
+**`DATABASE_USERNAME`**
-配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。
+配置`database`的`USERNAME`， 默认值 `root`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_PASSWORD`**
+
+配置`database`的`PASSWORD`， 默认值 `root`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_DATABASE`**
+
+配置`database`的`DATABASE`， 默认值 `dolphinscheduler`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_PARAMS`**
+
+配置`database`的`PARAMS`， 默认值 `characterEncoding=utf8`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl
index 14ea58b..0533c7e 100644
--- a/docker/build/conf/dolphinscheduler/common.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/common.properties.tpl
@@ -31,7 +31,7 @@ resource.storage.type=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE}
# HDFS
#============================================================================
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
-#resource.upload.path=/dolphinscheduler
+resource.upload.path=${RESOURCE_UPLOAD_PATH}
# whether kerberos starts
#hadoop.security.authentication.startup.state=false
@@ -61,13 +61,13 @@ kerberos.expire.time=7
fs.defaultFS=${DOLPHINSCHEDULER_FS_DEFAULTFS}
# if resource.storage.type=S3，s3 endpoint
-#fs.s3a.endpoint=http://192.168.199.91:9010
+fs.s3a.endpoint=${FS_S3A_ENDPOINT}
# if resource.storage.type=S3，s3 access key
-#fs.s3a.access.key=A3DXS30FO22544RE
+fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
# if resource.storage.type=S3，s3 secret key
-#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
+fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl
index aefb9e3..f7c5ee6 100644
--- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl
@@ -15,16 +15,11 @@
# limitations under the License.
#
-
-# mysql
-#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
-#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
-
-# postgre
-spring.datasource.driver-class-name=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
-spring.datasource.username=${POSTGRESQL_USERNAME}
-spring.datasource.password=${POSTGRESQL_PASSWORD}
+# db
+spring.datasource.driver-class-name=${DATABASE_DRIVER}
+spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}
+spring.datasource.username=${DATABASE_USERNAME}
+spring.datasource.password=${DATABASE_PASSWORD}
## base spring data source configuration todo need to remove
#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
index 362a4e8..51540aa 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
@@ -19,7 +19,7 @@
zookeeper.quorum=${ZOOKEEPER_QUORUM}
# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
+zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT}
# dolphinscheduler failover directory
#zookeeper.session.timeout=60000
diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh
index da6eb21..ac21fe4 100644
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@@ -24,22 +24,32 @@ echo "init env variables"
#============================================================================
# Database Source
#============================================================================
-export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"}
-export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"}
-export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"}
-export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"}
-export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"}
+export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"}
+export DATABASE_PORT=${DATABASE_PORT:-"5432"}
+export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"}
+export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
+export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
+export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
+export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
+export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
#============================================================================
# System
#============================================================================
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
+export DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE=${DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE:-"HDFS"}
+export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"}
+export DOLPHINSCHEDULER_FS_DEFAULTFS=${DOLPHINSCHEDULER_FS_DEFAULTFS:-"file:///data/dolphinscheduler"}
+export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
+export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
+export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
#============================================================================
# Zookeeper
#============================================================================
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
+export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
#============================================================================
# Master Server
diff --git a/docker/build/startup.sh b/docker/build/startup.sh
index 00bb9cd..0511788 100644
--- a/docker/build/startup.sh
+++ b/docker/build/startup.sh
@@ -22,24 +22,32 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
-# start postgresql
-initPostgreSQL() {
- echo "test postgresql service"
- while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+# start database
+initDatabase() {
+ echo "test ${DATABASE_TYPE} service"
+ while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 30 ]; then
- echo "Error: Couldn't connect to postgresql."
+ echo "Error: Couldn't connect to ${DATABASE_TYPE}."
exit 1
fi
- echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+ echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 5
done
- echo "connect postgresql service"
- v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1")
- if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
- echo "Error: Can't connect to database...${v}"
- exit 1
+ echo "connect ${DATABASE_TYPE} service"
+ if [ ${DATABASE_TYPE} = "mysql" ]; then
+ v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
+ if [ "$(echo ${v} | grep 'ERROR' | wc -l)" -eq 1 ]; then
+ echo "Error: Can't connect to database...${v}"
+ exit 1
+ fi
+ else
+ v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
+ if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then
+ echo "Error: Can't connect to database...${v}"
+ exit 1
+ fi
fi
echo "import sql data"
@@ -123,7 +131,7 @@ LOGFILE=/var/log/nginx/access.log
case "$1" in
(all)
initZK
- initPostgreSQL
+ initDatabase
initMasterServer
initWorkerServer
initApiServer
@@ -134,25 +142,25 @@ case "$1" in
;;
(master-server)
initZK
- initPostgreSQL
+ initDatabase
initMasterServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
;;
(worker-server)
initZK
- initPostgreSQL
+ initDatabase
initWorkerServer
initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
;;
(api-server)
initZK
- initPostgreSQL
+ initDatabase
initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
;;
(alert-server)
- initPostgreSQL
+ initDatabase
initAlertServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
;;
diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml
index 2034726..5ef116e 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -57,12 +57,13 @@ services:
- 12345:12345
environment:
TZ: Asia/Shanghai
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+ ZOOKEEPER_ROOT: /dolphinscheduler
DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS"
DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler"
healthcheck:
@@ -123,11 +124,11 @@ services:
ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: ""
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30s
@@ -156,12 +157,13 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+ ZOOKEEPER_ROOT: /dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
interval: 30s
@@ -192,12 +194,13 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+ ZOOKEEPER_ROOT: /dolphinscheduler
DOLPHINSCHEDULER_RESOURCE_STORAGE_TYPE: "HDFS"
DOLPHINSCHEDULER_FS_DEFAULTFS: "file:///data/dolphinscheduler"
healthcheck:
diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml
index 5c3873f..06dabce 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -58,11 +58,11 @@ services:
- 12345:12345
environment:
TZ: Asia/Shanghai
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
@@ -120,11 +120,11 @@ services:
ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: ""
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30
@@ -153,11 +153,11 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@@ -188,11 +188,11 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml
index 80e3edc..0264d8f 100644
--- a/dolphinscheduler-alert/pom.xml
+++ b/dolphinscheduler-alert/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-alert</artifactId>
<name>${project.artifactId}</name>
diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml
index e6dae1d..f0f53aa 100644
--- a/dolphinscheduler-api/pom.xml
+++ b/dolphinscheduler-api/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-api</artifactId>
<name>${project.artifactId}</name>
@@ -96,16 +96,24 @@
<artifactId>commons-collections</artifactId>
</dependency>
- <dependency>
- <groupId>org.quartz-scheduler</groupId>
- <artifactId>quartz</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>c3p0</artifactId>
- <groupId>c3p0</groupId>
- </exclusion>
- </exclusions>
- </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.mchange</groupId>
+ <artifactId>c3p0</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.mchange</groupId>
+ <artifactId>mchange-commons-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.zaxxer</groupId>
+ <artifactId>HikariCP-java6</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
index f5390e9..fd6c7af 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
@@ -217,7 +217,7 @@ public enum Status {
DATA_IS_NOT_VALID(50017,"data {0} not valid", "数据[{0}]无效"),
DATA_IS_NULL(50018,"data {0} is null", "数据[{0}]不能为空"),
PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle", "流程节点间存在循环依赖"),
- PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node %s parameter invalid", "流程节点[%s]参数无效"),
+ PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node {0} parameter invalid", "流程节点[{0}]参数无效"),
PROCESS_DEFINE_STATE_ONLINE(50021, "process definition {0} is already on line", "工作流定义[{0}]已上线"),
DELETE_PROCESS_DEFINE_BY_ID_ERROR(50022,"delete process definition by id error", "删除工作流定义错误"),
SCHEDULE_CRON_STATE_ONLINE(50023,"the status of schedule {0} is already on line", "调度配置[{0}]已上线"),
diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml
index b44897e..e41e5a0 100644
--- a/dolphinscheduler-common/pom.xml
+++ b/dolphinscheduler-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-common</artifactId>
<name>dolphinscheduler-common</name>
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index 56f3112..8af62a4 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -216,6 +216,11 @@ public final class Constants {
public static final String COLON = ":";
/**
+ * SPACE " "
+ */
+ public static final String SPACE = " ";
+
+ /**
* SINGLE_SLASH /
*/
public static final String SINGLE_SLASH = "/";
@@ -226,6 +231,15 @@ public final class Constants {
public static final String DOUBLE_SLASH = "//";
/**
+ * SINGLE_QUOTES "'"
+ */
+ public static final String SINGLE_QUOTES = "'";
+ /**
+ * DOUBLE_QUOTES "\""
+ */
+ public static final String DOUBLE_QUOTES = "\"";
+
+ /**
* SEMICOLON ;
*/
public static final String SEMICOLON = ";";
@@ -970,4 +984,14 @@ public final class Constants {
public static final int ABNORMAL_NODE_STATUS = 1;
+ /**
+ * exec shell scripts
+ */
+ public static final String SH = "sh";
+
+ /**
+ * pstree, get pud and sub pid
+ */
+ public static final String PSTREE = "pstree";
+
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java
similarity index 62%
copy from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java
copy to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java
index 3ad9aef..f1fde27 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java
@@ -14,31 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.service.queue;
+package org.apache.dolphinscheduler.common.enums;
+import com.baomidou.mybatisplus.annotation.EnumValue;
-public interface TaskPriorityQueue {
+public enum SqoopJobType {
+ CUSTOM(0, "CUSTOM"),
+ TEMPLATE(1, "TEMPLATE");
- /**
- * put task info
- *
- * @param taskInfo taskInfo
- * @throws Exception
- */
- void put(String taskInfo) throws Exception;
+ SqoopJobType(int code, String descp){
+ this.code = code;
+ this.descp = descp;
+ }
- /**
- * take taskInfo
- * @return taskInfo
- * @throws Exception
- */
- String take()throws Exception;
+ @EnumValue
+ private final int code;
+ private final String descp;
- /**
- * size
- *
- * @return size
- * @throws Exception
- */
- int size() throws Exception;
-}
\ No newline at end of file
+ public int getCode() {
+ return code;
+ }
+
+ public String getDescp() {
+ return descp;
+ }
+}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java
similarity index 63%
copy from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java
copy to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java
index 3ad9aef..4d279f5 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java
@@ -14,31 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.service.queue;
-
-
-public interface TaskPriorityQueue {
-
- /**
- * put task info
- *
- * @param taskInfo taskInfo
- * @throws Exception
- */
- void put(String taskInfo) throws Exception;
-
- /**
- * take taskInfo
- * @return taskInfo
- * @throws Exception
- */
- String take()throws Exception;
-
- /**
- * size
- *
- * @return size
- * @throws Exception
- */
- int size() throws Exception;
-}
\ No newline at end of file
+
+package org.apache.dolphinscheduler.common.enums;
+
+public enum SqoopQueryType {
+
+ FORM(0, "SQOOP_QUERY_FORM"),
+ SQL(1, "SQOOP_QUERY_SQL");
+
+ private final Integer code;
+
+ private final String desc;
+
+ SqoopQueryType(Integer code, String desc) {
+ this.code = code;
+ this.desc = desc;
+ }
+
+ public Integer getCode() {
+ return code;
+ }
+
+ public String getDesc() {
+ return desc;
+ }
+}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java
index f846b19..38c8347 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java
@@ -78,7 +78,7 @@ public abstract class AbstractShell {
/**
* If or not script finished executing
*/
- private volatile AtomicBoolean completed;
+ private AtomicBoolean completed;
public AbstractShell() {
this(0L);
@@ -128,7 +128,7 @@ public abstract class AbstractShell {
/**
* Run a command actual work
*/
- private void runCommand() throws IOException {
+ private void runCommand() throws IOException {
ProcessBuilder builder = new ProcessBuilder(getExecString());
Timer timeOutTimer = null;
ShellTimeoutTimerTask timeoutTimerTask = null;
@@ -153,11 +153,11 @@ public abstract class AbstractShell {
timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
}
final BufferedReader errReader =
- new BufferedReader(new InputStreamReader(process
- .getErrorStream()));
- BufferedReader inReader =
- new BufferedReader(new InputStreamReader(process
- .getInputStream()));
+ new BufferedReader(
+ new InputStreamReader(process.getErrorStream()));
+ BufferedReader inReader =
+ new BufferedReader(
+ new InputStreamReader(process.getInputStream()));
final StringBuilder errMsg = new StringBuilder();
// read error and input streams as this would free up the buffers
@@ -177,23 +177,35 @@ public abstract class AbstractShell {
}
}
};
+ Thread inThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ parseExecResult(inReader);
+ } catch (IOException ioe) {
+ logger.warn("Error reading the in stream", ioe);
+ }
+ super.run();
+ }
+ };
try {
errThread.start();
+ inThread.start();
} catch (IllegalStateException ise) { }
try {
// parse the output
- parseExecResult(inReader);
- exitCode = process.waitFor();
+ exitCode = process.waitFor();
try {
- // make sure that the error thread exits
+ // make sure that the error and in thread exits
errThread.join();
+ inThread.join();
} catch (InterruptedException ie) {
- logger.warn("Interrupted while reading the error stream", ie);
+ logger.warn("Interrupted while reading the error and in stream", ie);
}
- completed.set(true);
+ completed.compareAndSet(false,true);
//the timeout thread handling
//taken care in finally block
- if (exitCode != 0) {
+ if (exitCode != 0 || errMsg.length() > 0) {
throw new ExitCodeException(exitCode, errMsg.toString());
}
} catch (InterruptedException ie) {
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java
index 7f02f42..8b566a8 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java
@@ -16,6 +16,8 @@
*/
package org.apache.dolphinscheduler.common.task.sqoop;
+import org.apache.dolphinscheduler.common.enums.SqoopJobType;
+import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
@@ -29,6 +31,23 @@ import java.util.List;
public class SqoopParameters extends AbstractParameters {
/**
+ * sqoop job type:
+ * CUSTOM - custom sqoop job
+ * TEMPLATE - sqoop template job
+ */
+ private String jobType;
+
+ /**
+ * customJob eq 1, use customShell
+ */
+ private String customShell;
+
+ /**
+ * sqoop job name - map-reduce job name
+ */
+ private String jobName;
+
+ /**
* model type
*/
private String modelType;
@@ -53,6 +72,16 @@ public class SqoopParameters extends AbstractParameters {
*/
private String targetParams;
+ /**
+ * hadoop custom param for sqoop job
+ */
+ private List<Property> hadoopCustomParams;
+
+ /**
+ * sqoop advanced param
+ */
+ private List<Property> sqoopAdvancedParams;
+
public String getModelType() {
return modelType;
}
@@ -101,18 +130,74 @@ public class SqoopParameters extends AbstractParameters {
this.targetParams = targetParams;
}
+ public String getJobType() {
+ return jobType;
+ }
+
+ public void setJobType(String jobType) {
+ this.jobType = jobType;
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public String getCustomShell() {
+ return customShell;
+ }
+
+ public void setCustomShell(String customShell) {
+ this.customShell = customShell;
+ }
+
+ public List<Property> getHadoopCustomParams() {
+ return hadoopCustomParams;
+ }
+
+ public void setHadoopCustomParams(List<Property> hadoopCustomParams) {
+ this.hadoopCustomParams = hadoopCustomParams;
+ }
+
+ public List<Property> getSqoopAdvancedParams() {
+ return sqoopAdvancedParams;
+ }
+
+ public void setSqoopAdvancedParams(List<Property> sqoopAdvancedParams) {
+ this.sqoopAdvancedParams = sqoopAdvancedParams;
+ }
+
@Override
public boolean checkParameters() {
- return StringUtils.isNotEmpty(modelType)&&
- concurrency != 0 &&
- StringUtils.isNotEmpty(sourceType)&&
- StringUtils.isNotEmpty(targetType)&&
- StringUtils.isNotEmpty(sourceParams)&&
- StringUtils.isNotEmpty(targetParams);
+
+ boolean sqoopParamsCheck = false;
+
+ if (StringUtils.isEmpty(jobType)) {
+ return sqoopParamsCheck;
+ }
+
+ if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) {
+ sqoopParamsCheck = StringUtils.isEmpty(customShell) &&
+ StringUtils.isNotEmpty(modelType) &&
+ StringUtils.isNotEmpty(jobName) &&
+ concurrency != 0 &&
+ StringUtils.isNotEmpty(sourceType) &&
+ StringUtils.isNotEmpty(targetType) &&
+ StringUtils.isNotEmpty(sourceParams) &&
+ StringUtils.isNotEmpty(targetParams);
+ } else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) {
+ sqoopParamsCheck = StringUtils.isNotEmpty(customShell) &&
+ StringUtils.isEmpty(jobName);
+ }
+
+ return sqoopParamsCheck;
}
@Override
public List<ResourceInfo> getResourceFilesList() {
- return new ArrayList<>();
+ return new ArrayList<>();
}
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetHiveParameter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetHiveParameter.java
index f9bfde3..7ade555 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetHiveParameter.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetHiveParameter.java
@@ -41,6 +41,11 @@ public class TargetHiveParameter {
* hive overwrite
*/
private boolean hiveOverWrite;
+
+ /**
+ * hive target dir
+ */
+ private String hiveTargetDir;
/**
* replace delimiter
*/
@@ -117,4 +122,12 @@ public class TargetHiveParameter {
public void setHivePartitionValue(String hivePartitionValue) {
this.hivePartitionValue = hivePartitionValue;
}
+
+ public String getHiveTargetDir() {
+ return hiveTargetDir;
+ }
+
+ public void setHiveTargetDir(String hiveTargetDir) {
+ this.hiveTargetDir = hiveTargetDir;
+ }
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetMysqlParameter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetMysqlParameter.java
index 47126ae..86931cb 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetMysqlParameter.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/targets/TargetMysqlParameter.java
@@ -106,7 +106,7 @@ public class TargetMysqlParameter {
this.preQuery = preQuery;
}
- public boolean isUpdate() {
+ public boolean getIsUpdate() {
return isUpdate;
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index bae8f7f..de3d429 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -16,18 +16,32 @@
*/
package org.apache.dolphinscheduler.common.utils;
+import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
+import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS;
+import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
+import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import java.util.Optional;
+
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
-import java.nio.charset.Charset;
-import java.nio.charset.UnsupportedCharsetException;
-
-import static org.apache.dolphinscheduler.common.Constants.*;
-
/**
* file utils
*/
@@ -36,6 +50,8 @@ public class FileUtils {
public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler");
+ public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
+
/**
* get file suffix
*
@@ -118,7 +134,7 @@ public class FileUtils {
String fileName = String.format("%s/exec/process/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId),
Integer.toString(processDefineId), Integer.toString(processInstanceId));
File file = new File(fileName);
- if (!file.getParentFile().exists()){
+ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
@@ -138,24 +154,40 @@ public class FileUtils {
* @param userName user name
* @throws IOException errors
*/
- public static void createWorkDirAndUserIfAbsent(String execLocalPath, String userName) throws IOException{
+ public static void createWorkDirAndUserIfAbsent(String execLocalPath, String userName) throws IOException {
//if work dir exists, first delete
File execLocalPathFile = new File(execLocalPath);
- if (execLocalPathFile.exists()){
+ if (execLocalPathFile.exists()) {
org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
}
//create work dir
org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
- logger.info("create dir success {}" , execLocalPath);
-
+ String mkdirLog = "create dir success " + execLocalPath;
+ LoggerUtils.logInfo(Optional.ofNullable(logger), mkdirLog);
+ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), mkdirLog);
//if not exists this user,then create
- if (!OSUtils.getUserList().contains(userName)){
- OSUtils.createUser(userName);
+ OSUtils.taskLoggerThreadLocal.set(taskLoggerThreadLocal.get());
+ try {
+ if (!OSUtils.getUserList().contains(userName)) {
+ boolean isSuccessCreateUser = OSUtils.createUser(userName);
+
+ String infoLog;
+ if (isSuccessCreateUser) {
+ infoLog = String.format("create user name success %s", userName);
+ } else {
+ infoLog = String.format("create user name fail %s", userName);
+ }
+ LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog);
+ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog);
+ }
+ } catch (Throwable e) {
+ LoggerUtils.logError(Optional.ofNullable(logger), e);
+ LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
}
- logger.info("create user name success {}", userName);
+ OSUtils.taskLoggerThreadLocal.remove();
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
index 191df33..dfdff56 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
@@ -21,6 +21,7 @@ import org.slf4j.Logger;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -93,4 +94,24 @@ public class LoggerUtils {
}
return appIds;
}
+
+ public static void logError(Optional<Logger> optionalLogger
+ , String error) {
+ optionalLogger.ifPresent((Logger logger) -> logger.error(error));
+ }
+
+ public static void logError(Optional<Logger> optionalLogger
+ , Throwable e) {
+ optionalLogger.ifPresent((Logger logger) -> logger.error(e.getMessage(), e));
+ }
+
+ public static void logError(Optional<Logger> optionalLogger
+ , String error, Throwable e) {
+ optionalLogger.ifPresent((Logger logger) -> logger.error(error, e));
+ }
+
+ public static void logInfo(Optional<Logger> optionalLogger
+ , String info) {
+ optionalLogger.ifPresent((Logger logger) -> logger.info(info));
+ }
}
\ No newline at end of file
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
index 3505e59..133d993 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
@@ -16,9 +16,9 @@
*/
package org.apache.dolphinscheduler.common.utils;
+import org.apache.commons.configuration.Configuration;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
-import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import oshi.SystemInfo;
@@ -36,10 +36,7 @@ import java.math.RoundingMode;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
import java.util.regex.Pattern;
/**
@@ -50,6 +47,8 @@ public class OSUtils {
private static final Logger logger = LoggerFactory.getLogger(OSUtils.class);
+ public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
+
private static final SystemInfo SI = new SystemInfo();
public static final String TWO_DECIMAL = "0.00";
@@ -236,7 +235,9 @@ public class OSUtils {
try {
String userGroup = OSUtils.getGroup();
if (StringUtils.isEmpty(userGroup)) {
- logger.error("{} group does not exist for this operating system.", userGroup);
+ String errorLog = String.format("%s group does not exist for this operating system.", userGroup);
+ LoggerUtils.logError(Optional.ofNullable(logger), errorLog);
+ LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog);
return false;
}
if (isMacOS()) {
@@ -248,7 +249,8 @@ public class OSUtils {
}
return true;
} catch (Exception e) {
- logger.error(e.getMessage(), e);
+ LoggerUtils.logError(Optional.ofNullable(logger), e);
+ LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
}
return false;
@@ -261,10 +263,14 @@ public class OSUtils {
* @throws IOException in case of an I/O error
*/
private static void createLinuxUser(String userName, String userGroup) throws IOException {
- logger.info("create linux os user : {}", userName);
- String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
+ String infoLog1 = String.format("create linux os user : %s", userName);
+ LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
+ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
- logger.info("execute cmd : {}", cmd);
+ String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
+ String infoLog2 = String.format("execute cmd : %s", cmd);
+ LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
+ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
OSUtils.exeCmd(cmd);
}
@@ -275,13 +281,24 @@ public class OSUtils {
* @throws IOException in case of an I/O error
*/
private static void createMacUser(String userName, String userGroup) throws IOException {
- logger.info("create mac os user : {}", userName);
- String userCreateCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
- String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
- logger.info("create user command : {}", userCreateCmd);
- OSUtils.exeCmd(userCreateCmd);
- logger.info("append user to group : {}", appendGroupCmd);
+ Optional<Logger> optionalLogger = Optional.ofNullable(logger);
+ Optional<Logger> optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get());
+
+ String infoLog1 = String.format("create mac os user : %s", userName);
+ LoggerUtils.logInfo(optionalLogger, infoLog1);
+ LoggerUtils.logInfo(optionalTaskLogger, infoLog1);
+
+ String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
+ String infoLog2 = String.format("create user command : %s", createUserCmd);
+ LoggerUtils.logInfo(optionalLogger, infoLog2);
+ LoggerUtils.logInfo(optionalTaskLogger, infoLog2);
+ OSUtils.exeCmd(createUserCmd);
+
+ String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
+ String infoLog3 = String.format("append user to group : %s", appendGroupCmd);
+ LoggerUtils.logInfo(optionalLogger, infoLog3);
+ LoggerUtils.logInfo(optionalTaskLogger, infoLog3);
OSUtils.exeCmd(appendGroupCmd);
}
@@ -292,14 +309,20 @@ public class OSUtils {
* @throws IOException in case of an I/O error
*/
private static void createWindowsUser(String userName, String userGroup) throws IOException {
- logger.info("create windows os user : {}", userName);
- String userCreateCmd = String.format("net user \"%s\" /add", userName);
- String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
+ String infoLog1 = String.format("create windows os user : %s", userName);
+ LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
+ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
- logger.info("execute create user command : {}", userCreateCmd);
+ String userCreateCmd = String.format("net user \"%s\" /add", userName);
+ String infoLog2 = String.format("execute create user command : %s", userCreateCmd);
+ LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
+ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
OSUtils.exeCmd(userCreateCmd);
- logger.info("execute append user to group : {}", appendGroupCmd);
+ String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
+ String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd);
+ LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3);
+ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3);
OSUtils.exeCmd(appendGroupCmd);
}
@@ -338,22 +361,12 @@ public class OSUtils {
* @throws IOException errors
*/
public static String exeCmd(String command) throws IOException {
- BufferedReader br = null;
-
- try {
- Process p = Runtime.getRuntime().exec(command);
- br = new BufferedReader(new InputStreamReader(p.getInputStream()));
- String line;
- StringBuilder sb = new StringBuilder();
-
- while ((line = br.readLine()) != null) {
- sb.append(line + "\n");
- }
-
- return sb.toString();
- } finally {
- IOUtils.closeQuietly(br);
+ StringTokenizer st = new StringTokenizer(command);
+ String[] cmdArray = new String[st.countTokens()];
+ for (int i = 0; st.hasMoreTokens(); i++) {
+ cmdArray[i] = st.nextToken();
}
+ return exeShell(cmdArray);
}
/**
@@ -362,7 +375,7 @@ public class OSUtils {
* @return result of execute the shell
* @throws IOException errors
*/
- public static String exeShell(String command) throws IOException {
+ public static String exeShell(String[] command) throws IOException {
return ShellExecutor.execCommand(command);
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
index 84c60db..ee27e07 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
@@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
-import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType;
@@ -24,233 +24,255 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.sql.PreparedStatement;
-import java.text.ParseException;
-import java.util.*;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* parameter parse utils
*/
public class ParameterUtils {
- private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
-
- /**
- * convert parameters place holders
- *
- * @param parameterString parameter
- * @param parameterMap parameter map
- * @return convert parameters place holders
- */
- public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
- if (StringUtils.isEmpty(parameterString) || parameterMap == null) {
- return parameterString;
- }
+ private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
- //Get current time, schedule execute time
- String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
+ private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\]]+)]";
- Date cronTime = null;
+ private static final String DATE_START_PATTERN = "^[0-9]";
- if (StringUtils.isNotEmpty(cronTimeStr)) {
- try {
- cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
- } catch (ParseException e) {
- logger.error("parse {} exception", cronTimeStr, e);
- }
- } else {
- cronTime = new Date();
+ private ParameterUtils() {
+ throw new UnsupportedOperationException("Construct ParameterUtils");
}
- // replace variable ${} form,refers to the replacement of system variables and custom variables
- parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
-
- // replace time $[...] form, eg. $[yyyyMMdd]
- if (cronTime != null) {
- parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
+ /**
+ * convert parameters place holders
+ *
+ * @param parameterString parameter
+ * @param parameterMap parameter map
+ * @return convert parameters place holders
+ */
+ public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
+ if (StringUtils.isEmpty(parameterString)) {
+ return parameterString;
+ }
+ Date cronTime;
+ if (parameterMap != null && !parameterMap.isEmpty()) {
+ // replace variable ${} form,refers to the replacement of system variables and custom variables
+ parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+ }
+ if (parameterMap != null && null != parameterMap.get(Constants.PARAMETER_DATETIME)) {
+ //Get current time, schedule execute time
+ String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
+ cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
+ } else {
+ cronTime = new Date();
+ }
+ // replace time $[...] form, eg. $[yyyyMMdd]
+ if (cronTime != null) {
+ return dateTemplateParse(parameterString, cronTime);
+ }
+ return parameterString;
}
- return parameterString;
- }
-
- /**
- * new
- * convert parameters place holders
- *
- * @param parameterString parameter
- * @param parameterMap parameter map
- * @return convert parameters place holders
- */
- public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
- if (StringUtils.isEmpty(parameterString)) {
- return parameterString;
- }
- //Get current time, schedule execute time
- String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
- Date cronTime = null;
-
- if (StringUtils.isNotEmpty(cronTimeStr)) {
- try {
- cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
-
- } catch (ParseException e) {
- logger.error(String.format("parse %s exception", cronTimeStr), e);
- }
- } else {
- cronTime = new Date();
+ /**
+ * new
+ * convert parameters place holders
+ *
+ * @param parameterString parameter
+ * @param parameterMap parameter map
+ * @return convert parameters place holders
+ */
+ public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
+ if (StringUtils.isEmpty(parameterString)) {
+ return parameterString;
+ }
+ //Get current time, schedule execute time
+ String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
+ Date cronTime = null;
+
+ if (StringUtils.isNotEmpty(cronTimeStr)) {
+ cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
+
+ } else {
+ cronTime = new Date();
+ }
+
+ // replace variable ${} form,refers to the replacement of system variables and custom variables
+ parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+
+ // replace time $[...] form, eg. $[yyyyMMdd]
+ if (cronTime != null) {
+ return dateTemplateParse(parameterString, cronTime);
+ }
+ return parameterString;
}
- // replace variable ${} form,refers to the replacement of system variables and custom variables
- parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
+ /**
+ * set in parameter
+ *
+ * @param index index
+ * @param stmt preparedstatement
+ * @param dataType data type
+ * @param value value
+ * @throws Exception errors
+ */
+ public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception {
+ if (dataType.equals(DataType.VARCHAR)) {
+ stmt.setString(index, value);
+ } else if (dataType.equals(DataType.INTEGER)) {
+ stmt.setInt(index, Integer.parseInt(value));
+ } else if (dataType.equals(DataType.LONG)) {
+ stmt.setLong(index, Long.parseLong(value));
+ } else if (dataType.equals(DataType.FLOAT)) {
+ stmt.setFloat(index, Float.parseFloat(value));
+ } else if (dataType.equals(DataType.DOUBLE)) {
+ stmt.setDouble(index, Double.parseDouble(value));
+ } else if (dataType.equals(DataType.DATE)) {
+ stmt.setDate(index, java.sql.Date.valueOf(value));
+ } else if (dataType.equals(DataType.TIME)) {
+ stmt.setString(index, value);
+ } else if (dataType.equals(DataType.TIMESTAMP)) {
+ stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
+ } else if (dataType.equals(DataType.BOOLEAN)) {
+ stmt.setBoolean(index, Boolean.parseBoolean(value));
+ }
+ }
- // replace time $[...] form, eg. $[yyyyMMdd]
- if (cronTime != null) {
- parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
+ /**
+ * curing user define parameters
+ *
+ * @param globalParamMap global param map
+ * @param globalParamList global param list
+ * @param commandType command type
+ * @param scheduleTime schedule time
+ * @return curing user define parameters
+ */
+ public static String curingGlobalParams(Map<String, String> globalParamMap, List<Property> globalParamList,
+ CommandType commandType, Date scheduleTime) {
+
+ if (globalParamList == null || globalParamList.isEmpty()) {
+ return null;
+ }
+
+ Map<String, String> globalMap = new HashMap<>();
+ if (globalParamMap != null) {
+ globalMap.putAll(globalParamMap);
+ }
+ Map<String, String> allParamMap = new HashMap<>();
+ //If it is a complement, a complement time needs to be passed in, according to the task type
+ Map<String, String> timeParams = BusinessTimeUtils
+ .getBusinessTime(commandType, scheduleTime);
+ if (timeParams != null) {
+ allParamMap.putAll(timeParams);
+ }
+
+ allParamMap.putAll(globalMap);
+
+ Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
+
+ Map<String, String> resolveMap = new HashMap<>();
+ for (Map.Entry<String, String> entry : entries) {
+ String val = entry.getValue();
+ if (val.startsWith("$")) {
+ String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
+ resolveMap.put(entry.getKey(), str);
+ }
+ }
+ globalMap.putAll(resolveMap);
+
+ for (Property property : globalParamList) {
+ String val = globalMap.get(property.getProp());
+ if (val != null) {
+ property.setValue(val);
+ }
+ }
+ return JSONUtils.toJsonString(globalParamList);
}
- return parameterString;
- }
-
-
- /**
- * set in parameter
- * @param index index
- * @param stmt preparedstatement
- * @param dataType data type
- * @param value value
- * @throws Exception errors
- */
- public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value)throws Exception{
- if (dataType.equals(DataType.VARCHAR)){
- stmt.setString(index,value);
- }else if (dataType.equals(DataType.INTEGER)){
- stmt.setInt(index, Integer.parseInt(value));
- }else if (dataType.equals(DataType.LONG)){
- stmt.setLong(index, Long.parseLong(value));
- }else if (dataType.equals(DataType.FLOAT)){
- stmt.setFloat(index, Float.parseFloat(value));
- }else if (dataType.equals(DataType.DOUBLE)){
- stmt.setDouble(index, Double.parseDouble(value));
- }else if (dataType.equals(DataType.DATE)){
- stmt.setDate(index, java.sql.Date.valueOf(value));
- }else if (dataType.equals(DataType.TIME)){
- stmt.setString(index, value);
- }else if (dataType.equals(DataType.TIMESTAMP)){
- stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
- }else if (dataType.equals(DataType.BOOLEAN)){
- stmt.setBoolean(index,Boolean.parseBoolean(value));
- }
- }
-
- /**
- * curing user define parameters
- *
- * @param globalParamMap global param map
- * @param globalParamList global param list
- * @param commandType command type
- * @param scheduleTime schedule time
- * @return curing user define parameters
- */
- public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList,
- CommandType commandType, Date scheduleTime){
-
- if (globalParamList == null || globalParamList.isEmpty()) {
- return null;
+
+ /**
+ * handle escapes
+ *
+ * @param inputString input string
+ * @return string filter escapes
+ */
+ public static String handleEscapes(String inputString) {
+
+ if (StringUtils.isNotEmpty(inputString)) {
+ return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
+ }
+ return inputString;
}
- Map<String, String> globalMap = new HashMap<>();
- if (globalParamMap!= null){
- globalMap.putAll(globalParamMap);
+ /**
+ * $[yyyyMMdd] replace schedule time
+ */
+ public static String replaceScheduleTime(String text, Date scheduleTime) {
+ Map<String, Property> paramsMap = new HashMap<>();
+ //if getScheduleTime null ,is current date
+ if (null == scheduleTime) {
+ scheduleTime = new Date();
+ }
+
+ String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
+ Property p = new Property();
+ p.setValue(dateTime);
+ p.setProp(Constants.PARAMETER_SHECDULE_TIME);
+ paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
+ text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
+
+ return text;
}
- Map<String,String> allParamMap = new HashMap<>();
- //If it is a complement, a complement time needs to be passed in, according to the task type
- Map<String,String> timeParams = BusinessTimeUtils
- .getBusinessTime(commandType, scheduleTime);
- if (timeParams != null) {
- allParamMap.putAll(timeParams);
+ /**
+ * format convert
+ *
+ * @param paramsMap params map
+ * @return Map of converted
+ * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
+ */
+ public static Map<String, String> convert(Map<String, Property> paramsMap) {
+ Map<String, String> map = new HashMap<>();
+ Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
+ while (iter.hasNext()) {
+ Map.Entry<String, Property> en = iter.next();
+ map.put(en.getKey(), en.getValue().getValue());
+ }
+ return map;
}
- allParamMap.putAll(globalMap);
+ private static String dateTemplateParse(String templateStr, Date date) {
+ if (templateStr == null) {
+ return null;
+ }
+ Pattern pattern = Pattern.compile(DATE_PARSE_PATTERN);
- Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
+ StringBuffer newValue = new StringBuffer(templateStr.length());
- Map<String,String> resolveMap = new HashMap<>();
- for (Map.Entry<String,String> entry : entries){
- String val = entry.getValue();
- if (val.startsWith("$")){
- String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
- resolveMap.put(entry.getKey(),str);
- }
- }
- globalMap.putAll(resolveMap);
-
- for (Property property : globalParamList){
- String val = globalMap.get(property.getProp());
- if (val != null){
- property.setValue(val);
- }
- }
- return JSON.toJSONString(globalParamList);
- }
+ Matcher matcher = pattern.matcher(templateStr);
+ while (matcher.find()) {
+ String key = matcher.group(1);
+ if (Pattern.matches(DATE_START_PATTERN, key)) {
+ continue;
+ }
+ String value = TimePlaceholderUtils.getPlaceHolderTime(key, date);
+ assert value != null;
+ matcher.appendReplacement(newValue, value);
+ }
- /**
- * handle escapes
- * @param inputString input string
- * @return string filter escapes
- */
- public static String handleEscapes(String inputString){
+ matcher.appendTail(newValue);
- if(StringUtils.isNotEmpty(inputString)){
- return inputString.replace("%", "////%");
+ return newValue.toString();
}
- return inputString;
- }
-
-
- /**
- * $[yyyyMMdd] replace schedule time
- * @param text
- * @param scheduleTime
- * @return
- */
- public static String replaceScheduleTime(String text, Date scheduleTime) {
- Map<String, Property> paramsMap = new HashMap<>();
- //if getScheduleTime null ,is current date
- if (null == scheduleTime) {
- scheduleTime = new Date();
- }
-
- String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
- Property p = new Property();
- p.setValue(dateTime);
- p.setProp(Constants.PARAMETER_SHECDULE_TIME);
- paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
- text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
-
- return text;
- }
-
-
- /**
- * format convert
- * @param paramsMap params map
- * @return Map of converted
- * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
- */
- public static Map<String,String> convert(Map<String,Property> paramsMap){
- Map<String,String> map = new HashMap<>();
- Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
- while (iter.hasNext()){
- Map.Entry<String, Property> en = iter.next();
- map.put(en.getKey(),en.getValue().getValue());
- }
- return map;
- }
+
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
index af2817a..256f199 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
@@ -14,11 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
public class StringUtils {
+
public static final String EMPTY = "";
+ private StringUtils() {
+ throw new UnsupportedOperationException("Construct StringUtils");
+ }
+
public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0;
}
@@ -27,14 +33,26 @@ public class StringUtils {
return !isEmpty(cs);
}
- public static boolean isBlank(String s){
+ public static boolean isBlank(String s) {
if (isEmpty(s)) {
return true;
}
return s.trim().length() == 0;
}
- public static boolean isNotBlank(String s){
+ public static boolean isNotBlank(String s) {
return !isBlank(s);
}
+
+ public static String replaceNRTtoUnderline(String src) {
+ if (isBlank(src)) {
+ return src;
+ } else {
+ return src.replaceAll("[\n|\r|\t]", "_");
+ }
+ }
+
+ public static String trim(String str) {
+ return str == null ? null : str.trim();
+ }
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
index 35cb018..6a46bbe 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
@@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils.placeholder;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,8 +47,8 @@ public class TimePlaceholderUtils {
* Replaces all placeholders of format {@code ${name}} with the value returned
* from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}.
*
- * @param value the value containing the placeholders to be replaced
- * @param date custom date
+ * @param value the value containing the placeholders to be replaced
+ * @param date custom date
* @param ignoreUnresolvablePlaceholders ignore unresolvable placeholders
* @return the supplied value with placeholders replaced inline
*/
@@ -59,11 +60,11 @@ public class TimePlaceholderUtils {
return helper.replacePlaceholders(value, new TimePlaceholderResolver(value, date));
}
-
/**
* Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix.
+ *
* @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should
- * be ignored ({@code true}) or cause an exception ({@code false})
+ * be ignored ({@code true}) or cause an exception ({@code false})
*/
private static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) {
return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders);
@@ -89,7 +90,7 @@ public class TimePlaceholderUtils {
* Change the sign in the expression to P (positive) N (negative)
*
* @param expression
- * @return eg. "-3+-6*(+8)-(-5) -> S3+S6*(P8)-(S5)"
+ * @return eg. "-3+-6*(+8)-(-5) -> S3+S6*(P8)-(S5)"
*/
private static String convert(String expression) {
char[] arr = expression.toCharArray();
@@ -262,7 +263,7 @@ public class TimePlaceholderUtils {
* Placeholder replacement resolver
*/
private static class TimePlaceholderResolver implements
- PropertyPlaceholderHelper.PlaceholderResolver {
+ PropertyPlaceholderHelper.PlaceholderResolver {
private final String value;
@@ -278,12 +279,28 @@ public class TimePlaceholderUtils {
try {
return calculateTime(placeholderName, date);
} catch (Exception ex) {
- logger.error("resolve placeholder '{}' in [ {} ]" , placeholderName, value, ex);
+ logger.error("resolve placeholder '{}' in [ {} ]", placeholderName, value, ex);
return null;
}
}
}
+ /**
+ * return the formatted date according to the corresponding date format
+ *
+ * @param expression date expression
+ * @param date date
+ * @return reformat date
+ */
+ public static String getPlaceHolderTime(String expression, Date date) {
+ if (StringUtils.isBlank(expression)) {
+ return null;
+ }
+ if (null == date) {
+ return null;
+ }
+ return calculateTime(expression, date);
+ }
/**
* calculate time
@@ -320,9 +337,10 @@ public class TimePlaceholderUtils {
/**
* calculate time expresstion
+ *
* @param expression expresstion
- * @param date date
- * @return map with date, date format
+ * @param date date
+ * @return map with date, date format
*/
public static Map.Entry<Date, String> calcTimeExpression(String expression, Date date) {
Map.Entry<Date, String> resultEntry;
@@ -346,8 +364,9 @@ public class TimePlaceholderUtils {
/**
* get first day of month
+ *
* @param expression expresstion
- * @param date date
+ * @param date date
* @return first day of month
*/
public static Map.Entry<Date, String> calcMonthBegin(String expression, Date date) {
@@ -369,8 +388,9 @@ public class TimePlaceholderUtils {
/**
* get last day of month
+ *
* @param expression expresstion
- * @param date date
+ * @param date date
* @return last day of month
*/
public static Map.Entry<Date, String> calcMonthEnd(String expression, Date date) {
@@ -392,8 +412,9 @@ public class TimePlaceholderUtils {
/**
* get first day of week
+ *
* @param expression expresstion
- * @param date date
+ * @param date date
* @return monday
*/
public static Map.Entry<Date, String> calcWeekStart(String expression, Date date) {
@@ -414,8 +435,9 @@ public class TimePlaceholderUtils {
/**
* get last day of week
+ *
* @param expression expresstion
- * @param date date
+ * @param date date
* @return last day of week
*/
public static Map.Entry<Date, String> calcWeekEnd(String expression, Date date) {
@@ -437,8 +459,9 @@ public class TimePlaceholderUtils {
/**
* calc months expression
+ *
* @param expression expresstion
- * @param date date
+ * @param date date
* @return calc months
*/
public static Map.Entry<Date, String> calcMonths(String expression, Date date) {
@@ -461,7 +484,7 @@ public class TimePlaceholderUtils {
* calculate time expression
*
* @param expression expresstion
- * @param date date
+ * @param date date
* @return calculate time expression with date,format
*/
public static Map.Entry<Date, String> calcMinutes(String expression, Date date) {
@@ -471,7 +494,7 @@ public class TimePlaceholderUtils {
if (Character.isDigit(expression.charAt(index + 1))) {
String addMinuteExpr = expression.substring(index + 1);
Date targetDate = org.apache.commons.lang.time.DateUtils
- .addMinutes(date, calcMinutes(addMinuteExpr));
+ .addMinutes(date, calcMinutes(addMinuteExpr));
String dateFormat = expression.substring(0, index);
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat);
@@ -482,7 +505,7 @@ public class TimePlaceholderUtils {
if (Character.isDigit(expression.charAt(index + 1))) {
String addMinuteExpr = expression.substring(index + 1);
Date targetDate = org.apache.commons.lang.time.DateUtils
- .addMinutes(date, 0 - calcMinutes(addMinuteExpr));
+ .addMinutes(date, 0 - calcMinutes(addMinuteExpr));
String dateFormat = expression.substring(0, index);
return new AbstractMap.SimpleImmutableEntry<>(targetDate, dateFormat);
@@ -512,7 +535,7 @@ public class TimePlaceholderUtils {
} else {
calcExpression = String.format("60*24*(%s)%s", minuteExpression.substring(0, index),
- minuteExpression.substring(index));
+ minuteExpression.substring(index));
}
return calculate(calcExpression);
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
index 7106804..902e3cf 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
@@ -64,7 +64,11 @@ public class OSUtilsTest {
@Test
public void createUser() {
boolean result = OSUtils.createUser("test123");
- Assert.assertTrue(result);
+ if (result) {
+ Assert.assertTrue("create user test123 success", true);
+ } else {
+ Assert.assertTrue("create user test123 fail", true);
+ }
}
@Test
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
index abdc15c..796a5fa 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
@@ -14,23 +14,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils;
-import com.alibaba.fastjson.JSON;
-import org.apache.commons.lang.time.DateUtils;
+import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders;
+
+import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME;
-import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders;
-
public class ParameterUtilsTest {
public static final Logger logger = LoggerFactory.getLogger(ParameterUtilsTest.class);
@@ -39,13 +45,13 @@ public class ParameterUtilsTest {
* Test convertParameterPlaceholders
*/
@Test
- public void testConvertParameterPlaceholders() throws Exception {
+ public void testConvertParameterPlaceholders() throws ParseException {
// parameterString,parameterMap is null
Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, null));
// parameterString is null,parameterMap is not null
- Map<String, String> parameterMap = new HashMap<String,String>();
- parameterMap.put("testParameter","testParameter");
+ Map<String, String> parameterMap = new HashMap<String, String>();
+ parameterMap.put("testParameter", "testParameter");
Assert.assertNull(ParameterUtils.convertParameterPlaceholders(null, parameterMap));
// parameterString、parameterMap is not null
@@ -53,67 +59,79 @@ public class ParameterUtilsTest {
Assert.assertEquals(parameterString, ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap));
//replace variable ${} form
- parameterMap.put("testParameter2","${testParameter}");
- Assert.assertEquals(parameterString,PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true));
+ parameterMap.put("testParameter2", "${testParameter}");
+ Assert.assertEquals(parameterString, PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true));
// replace time $[...] form, eg. $[yyyyMMdd]
Date cronTime = new Date();
Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTime, true));
// replace time $[...] form, eg. $[yyyyMMdd]
- Date cronTimeStr = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME});
+ Date cronTimeStr = DateUtils.stringToDate("2019-02-02 00:00:00");
Assert.assertEquals(parameterString, replacePlaceholders(parameterString, cronTimeStr, true));
}
+ @Test
+ public void testConvertParameterPlaceholders2() {
+ String parameterString =
+ "${user} is userName, '$[1]' '$[add_months(yyyyMMdd,12*2)]' '$[add_months(yyyyMMdd,-12*2)]' '$[add_months(yyyyMMdd,3)]' '$[add_months(yyyyMMdd,-4)]' "
+ + "'$[yyyyMMdd+7*2]' '$[yyyyMMdd-7*2]' '$[yyyyMMdd+3]' '$[0]' '$[yyyyMMdd-3]' '$[HHmmss+2/24]' '$[HHmmss-1/24]' '$[HHmmss+3/24/60]' '$[HHmmss-2/24/60]' '$[3]'";
+ Map<String, String> parameterMap = new HashMap<>();
+ parameterMap.put("user", "Kris");
+ parameterMap.put(Constants.PARAMETER_DATETIME, "20201201123000");
+ parameterString = ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap);
+ Assert.assertEquals("Kris is userName, '$[1]' '20221201' '20181201' '20210301' '20200801' '20201215' '20201117' '20201204' '$[0]' '20201128' '143000' '113000' '123300' '122800' '$[3]'",
+ parameterString);
+ }
+
/**
* Test curingGlobalParams
*/
@Test
- public void testCuringGlobalParams() throws Exception {
+ public void testCuringGlobalParams() {
//define globalMap
Map<String, String> globalParamMap = new HashMap<>();
- globalParamMap.put("globalParams1","Params1");
+ globalParamMap.put("globalParams1", "Params1");
//define globalParamList
List<Property> globalParamList = new ArrayList<>();
//define scheduleTime
- Date scheduleTime = DateUtils.parseDate("20191220145900", new String[]{PARAMETER_FORMAT_TIME});
+ Date scheduleTime = DateUtils.stringToDate("2019-12-20 00:00:00");
//test globalParamList is null
String result = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertNull(result);
- Assert.assertNull(ParameterUtils.curingGlobalParams(null,null,CommandType.START_CURRENT_TASK_PROCESS,null));
- Assert.assertNull(ParameterUtils.curingGlobalParams(globalParamMap,null,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime));
+ Assert.assertNull(ParameterUtils.curingGlobalParams(null, null, CommandType.START_CURRENT_TASK_PROCESS, null));
+ Assert.assertNull(ParameterUtils.curingGlobalParams(globalParamMap, null, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime));
//test globalParamList is not null
- Property property=new Property("testGlobalParam", Direct.IN, DataType.VARCHAR,"testGlobalParam");
+ Property property = new Property("testGlobalParam", Direct.IN, DataType.VARCHAR, "testGlobalParam");
globalParamList.add(property);
- String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime);
- Assert.assertEquals(result2, JSON.toJSONString(globalParamList));
+ String result2 = ParameterUtils.curingGlobalParams(null, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
+ Assert.assertEquals(result2, JSONUtils.toJsonString(globalParamList));
- String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null);
- Assert.assertEquals(result3, JSON.toJSONString(globalParamList));
+ String result3 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, null);
+ Assert.assertEquals(result3, JSONUtils.toJsonString(globalParamList));
String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
- Assert.assertEquals(result4, JSON.toJSONString(globalParamList));
+ Assert.assertEquals(result4, JSONUtils.toJsonString(globalParamList));
//test var $ startsWith
- globalParamMap.put("bizDate","${system.biz.date}");
- globalParamMap.put("b1zCurdate","${system.biz.curdate}");
-
+ globalParamMap.put("bizDate", "${system.biz.date}");
+ globalParamMap.put("b1zCurdate", "${system.biz.curdate}");
- Property property2=new Property("testParamList1", Direct.IN, DataType.VARCHAR,"testParamList");
- Property property3=new Property("testParamList2", Direct.IN, DataType.VARCHAR,"{testParamList1}");
- Property property4=new Property("testParamList3", Direct.IN, DataType.VARCHAR,"${b1zCurdate}");
+ Property property2 = new Property("testParamList1", Direct.IN, DataType.VARCHAR, "testParamList");
+ Property property3 = new Property("testParamList2", Direct.IN, DataType.VARCHAR, "{testParamList1}");
+ Property property4 = new Property("testParamList3", Direct.IN, DataType.VARCHAR, "${b1zCurdate}");
globalParamList.add(property2);
globalParamList.add(property3);
globalParamList.add(property4);
String result5 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
- Assert.assertEquals(result5,JSONUtils.toJsonString(globalParamList));
+ Assert.assertEquals(result5, JSONUtils.toJsonString(globalParamList));
}
/**
@@ -122,9 +140,9 @@ public class ParameterUtilsTest {
@Test
public void testHandleEscapes() throws Exception {
Assert.assertNull(ParameterUtils.handleEscapes(null));
- Assert.assertEquals("",ParameterUtils.handleEscapes(""));
- Assert.assertEquals("test Parameter",ParameterUtils.handleEscapes("test Parameter"));
- Assert.assertEquals("////%test////%Parameter",ParameterUtils.handleEscapes("%test%Parameter"));
+ Assert.assertEquals("", ParameterUtils.handleEscapes(""));
+ Assert.assertEquals("test Parameter", ParameterUtils.handleEscapes("test Parameter"));
+ Assert.assertEquals("////%test////%Parameter", ParameterUtils.handleEscapes("%test%Parameter"));
}
}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java
index d204dfd..1037b9b 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.common.utils.placeholder;
import org.apache.dolphinscheduler.common.utils.DateUtils;
@@ -25,44 +26,49 @@ import java.util.Date;
public class TimePlaceholderUtilsTest {
- Date date = null;
+ private Date date;
@Before
- public void init(){
- date = DateUtils.parse("20170101010101","yyyyMMddHHmmss");
+ public void init() {
+ date = DateUtils.parse("20170101010101", "yyyyMMddHHmmss");
+ }
+
+ @Test
+ public void replacePlaceholdersT() {
+ Assert.assertEquals("2017test12017:***2016-12-31,20170102,20170130,20161227,20161231", TimePlaceholderUtils
+ .replacePlaceholders("$[yyyy]test1$[yyyy:***]$[yyyy-MM-dd-1],$[month_begin(yyyyMMdd, 1)],$[month_end(yyyyMMdd, -1)],$[week_begin(yyyyMMdd, 1)],$[week_end(yyyyMMdd, -1)]",
+ date, true));
+
+ Assert.assertEquals("1483200061,1483290061,1485709261,1482771661,1483113600,1483203661", TimePlaceholderUtils.replacePlaceholders("$[timestamp(yyyyMMdd00mmss)],"
+ + "$[timestamp(month_begin(yyyyMMddHHmmss, 1))],"
+ + "$[timestamp(month_end(yyyyMMddHHmmss, -1))],"
+ + "$[timestamp(week_begin(yyyyMMddHHmmss, 1))],"
+ + "$[timestamp(week_end(yyyyMMdd000000, -1))],"
+ + "$[timestamp(yyyyMMddHHmmss)]",
+ date, true));
+ }
+
+ @Test
+ public void calcMinutesT() {
+ Assert.assertEquals("Sun Jan 01 01:01:01 CST 2017=yyyy", TimePlaceholderUtils.calcMinutes("yyyy", date).toString());
+ Assert.assertEquals("Sun Jan 08 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+7*1", date).toString());
+ Assert.assertEquals("Sun Dec 25 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-7*1", date).toString());
+ Assert.assertEquals("Mon Jan 02 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+1", date).toString());
+ Assert.assertEquals("Sat Dec 31 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-1", date).toString());
+ Assert.assertEquals("Sun Jan 01 02:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH+1/24", date).toString());
+ Assert.assertEquals("Sun Jan 01 00:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH-1/24", date).toString());
+ }
+
+ @Test
+ public void calcMonthsT() {
+ Assert.assertEquals("Mon Jan 01 01:01:01 CST 2018=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,12*1)", date).toString());
+ Assert.assertEquals("Fri Jan 01 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,-12*1)", date).toString());
}
-// @Test
-// public void replacePlaceholdersT() {
-// Assert.assertEquals("2017test12017:***2016-12-31,20170102,20170130,20161227,20161231", TimePlaceholderUtils.replacePlaceholders("$[yyyy]test1$[yyyy:***]$[yyyy-MM-dd-1],$[month_begin(yyyyMMdd, 1)],$[month_end(yyyyMMdd, -1)],$[week_begin(yyyyMMdd, 1)],$[week_end(yyyyMMdd, -1)]",
-// date, true));
-//
-// Assert.assertEquals("1483200061,1483290061,1485709261,1482771661,1483113600,1483203661", TimePlaceholderUtils.replacePlaceholders("$[timestamp(yyyyMMdd00mmss)],"
-// + "$[timestamp(month_begin(yyyyMMddHHmmss, 1))],"
-// + "$[timestamp(month_end(yyyyMMddHHmmss, -1))],"
-// + "$[timestamp(week_begin(yyyyMMddHHmmss, 1))],"
-// + "$[timestamp(week_end(yyyyMMdd000000, -1))],"
-// + "$[timestamp(yyyyMMddHHmmss)]",
-// date, true));
-// }
-//
-//
-//
-// @Test
-// public void calcMinutesT() {
-// Assert.assertEquals("Sun Jan 01 01:01:01 CST 2017=yyyy", TimePlaceholderUtils.calcMinutes("yyyy", date).toString());
-// Assert.assertEquals("Sun Jan 08 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+7*1", date).toString());
-// Assert.assertEquals("Sun Dec 25 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-7*1", date).toString());
-// Assert.assertEquals("Mon Jan 02 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+1", date).toString());
-// Assert.assertEquals("Sat Dec 31 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-1", date).toString());
-// Assert.assertEquals("Sun Jan 01 02:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH+1/24", date).toString());
-// Assert.assertEquals("Sun Jan 01 00:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH-1/24", date).toString());
-// }
-//
-// @Test
-// public void calcMonthsT() {
-// Assert.assertEquals("Mon Jan 01 01:01:01 CST 2018=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,12*1)", date).toString());
-// Assert.assertEquals("Fri Jan 01 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,-12*1)", date).toString());
-// }
+ @Test
+ public void testGetPlaceHolderTime() {
+
+ Assert.assertEquals("20170101", TimePlaceholderUtils.getPlaceHolderTime("yyyyMMdd", date));
+ }
}
\ No newline at end of file
diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml
index c622753..7aaabab 100644
--- a/dolphinscheduler-dao/pom.xml
+++ b/dolphinscheduler-dao/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-dao</artifactId>
<name>${project.artifactId}</name>
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java
index b2daae2..72eb90d 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java
@@ -119,6 +119,8 @@ public class DolphinSchedulerManager {
upgradeDao.upgradeDolphinSchedulerWorkerGroup();
} else if ("1.3.2".equals(schemaVersion)) {
upgradeDao.upgradeDolphinSchedulerResourceList();
+ } else if ("1.3.4".equals(schemaVersion)) {
+ upgradeDao.upgradeDolphinSchedulerSqoopTaskParams();
}
version = schemaVersion;
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
index d111a8c..5413fd3 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
@@ -530,4 +530,50 @@ public abstract class UpgradeDao extends AbstractBaseDao {
}
+ /**
+ * upgrade DolphinScheduler sqoop task params
+ * ds-1.3.4 modify the sqoop task params for process definition json
+ */
+ public void upgradeDolphinSchedulerSqoopTaskParams() {
+ upgradeProcessDefinitionJsonSqoopTaskParams();
+ }
+
+ /**
+ * upgradeProcessDefinitionJsonSqoopTaskParams
+ */
+ protected void upgradeProcessDefinitionJsonSqoopTaskParams() {
+ ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao();
+ Map<Integer,String> replaceProcessDefinitionMap = new HashMap<>();
+
+ try {
+ Map<Integer,String> processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection());
+ for (Map.Entry<Integer,String> entry : processDefinitionJsonMap.entrySet()) {
+ JSONObject jsonObject = JSONObject.parseObject(entry.getValue());
+ JSONArray tasks = JSONArray.parseArray(jsonObject.getString("tasks"));
+
+ for (int i = 0; i < tasks.size(); i++) {
+ JSONObject task = tasks.getJSONObject(i);
+ String taskType = task.getString("type");
+ if ("SQOOP".equals(taskType) && !task.getString("params").contains("jobType")) {
+ JSONObject taskParams = JSONObject.parseObject(task.getString("params"));
+ taskParams.put("jobType","TEMPLATE");
+ taskParams.put("jobName","sqoop-job");
+ taskParams.put("hadoopCustomParams", new JSONArray());
+ taskParams.put("sqoopAdvancedParams", new JSONArray());
+
+ task.remove(task.getString("params"));
+ task.put("params",taskParams);
+ }
+ jsonObject.remove(jsonObject.getString("tasks"));
+ jsonObject.put("tasks", tasks);
+ replaceProcessDefinitionMap.put(entry.getKey(), jsonObject.toJSONString());
+ }
+ }
+ if (replaceProcessDefinitionMap.size() > 0) {
+ processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),replaceProcessDefinitionMap);
+ }
+ } catch (Exception e) {
+ logger.error("update process definition json sqoop task params error: {}", e.getMessage());
+ }
+ }
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java
index e2da27d..aa64e53 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java
@@ -256,7 +256,7 @@ public class DagHelper {
}
for (String dependNodeName : dependList) {
TaskNode dependNode = dag.getNode(dependNodeName);
- if (completeTaskList.containsKey(dependNodeName)
+ if (dependNode == null || completeTaskList.containsKey(dependNodeName)
|| dependNode.isForbidden()
|| skipTaskNodeList.containsKey(dependNodeName)) {
continue;
@@ -374,11 +374,14 @@ public class DagHelper {
DAG<String, TaskNode, TaskNodeRelation> dag,
Map<String, TaskInstance> completeTaskList,
Map<String, TaskNode> skipTaskNodeList){
+ if (!dag.containsNode(skipNodeName)) {
+ return;
+ }
skipTaskNodeList.putIfAbsent(skipNodeName, dag.getNode(skipNodeName));
Collection<String> postNodeList = dag.getSubsequentNodes(skipNodeName);
- for(String post : postNodeList){
+ for (String post : postNodeList) {
TaskNode postNode = dag.getNode(post);
- if(isTaskNodeNeedSkip(postNode, skipTaskNodeList)){
+ if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) {
setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList);
}
}
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml
index 3559ca9..1ab759b 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml
@@ -66,11 +66,12 @@
</select>
<select id="queryProcessInstanceListPaging" resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
- select instance.*
+ select instance.id, instance.process_definition_id, instance.command_type, instance.executor_id,
+ instance.name, instance.state, instance.schedule_time, instance.start_time, instance.end_time,
+ instance.run_times, instance.recovery, instance.host
from t_ds_process_instance instance
join t_ds_process_definition define ON instance.process_definition_id = define.id
- where 1=1
- and instance.is_sub_process=0
+ where instance.is_sub_process=0
and define.project_id = #{projectId}
<if test="processDefinitionId != 0">
and instance.process_definition_id = #{processDefinitionId}
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
index 143761b..0e03abd 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
@@ -100,8 +100,8 @@
<select id="queryTaskInstanceListPaging" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
select instance.*,process.name as process_instance_name
from t_ds_task_instance instance
- join t_ds_process_definition define ON instance.process_definition_id = define.id
- join t_ds_process_instance process on process.id=instance.process_instance_id
+ left join t_ds_process_definition define on instance.process_definition_id = define.id
+ left join t_ds_process_instance process on process.id=instance.process_instance_id
where define.project_id = #{projectId}
<if test="startTime != null">
and instance.start_time > #{startTime} and instance.start_time <![CDATA[ <=]]> #{endTime}
diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml
index 7b577e0..ae48319 100644
--- a/dolphinscheduler-dist/pom.xml
+++ b/dolphinscheduler-dist/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE
index 1acee57..71d3f9a 100644
--- a/dolphinscheduler-dist/release-docs/LICENSE
+++ b/dolphinscheduler-dist/release-docs/LICENSE
@@ -287,10 +287,10 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
httpclient 4.4.1: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient/4.4.1, Apache 2.0
httpcore 4.4.1: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore/4.4.1, Apache 2.0
httpmime 4.5.7: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime/4.5.7, Apache 2.0
- jackson-annotations 2.9.8: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.9.8, Apache 2.0
- jackson-core 2.9.8: https://github.com/FasterXML/jackson-core, Apache 2.0
+ jackson-annotations 2.9.10: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.9.10, Apache 2.0
+ jackson-core 2.9.10: https://github.com/FasterXML/jackson-core, Apache 2.0
jackson-core-asl 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-core-asl/1.9.13, Apache 2.0
- jackson-databind 2.9.8: https://github.com/FasterXML/jackson-databind, Apache 2.0
+ jackson-databind 2.9.10: https://github.com/FasterXML/jackson-databind, Apache 2.0
jackson-datatype-jdk8 2.9.8: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.9.8, Apache 2.0
jackson-datatype-jsr310 2.9.8: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.9.8, Apache 2.0
jackson-jaxrs 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-jaxrs/1.9.13, Apache 2.0 and LGPL 2.1
@@ -343,8 +343,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
opencsv 2.3: https://mvnrepository.com/artifact/net.sf.opencsv/opencsv/2.3, Apache 2.0
parquet-hadoop-bundle 1.8.1: https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop-bundle/1.8.1, Apache 2.0
poi 3.17: https://mvnrepository.com/artifact/org.apache.poi/poi/3.17, Apache 2.0
- quartz 2.2.3: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.2.3, Apache 2.0
- quartz-jobs 2.2.3: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.2.3, Apache 2.0
+ quartz 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz/2.3.0, Apache 2.0
+ quartz-jobs 2.3.0: https://mvnrepository.com/artifact/org.quartz-scheduler/quartz-jobs/2.3.0, Apache 2.0
snakeyaml 1.23: https://mvnrepository.com/artifact/org.yaml/snakeyaml/1.23, Apache 2.0
snappy 0.2: https://mvnrepository.com/artifact/org.iq80.snappy/snappy/0.2, Apache 2.0
snappy-java 1.0.4.1: https://github.com/xerial/snappy-java, Apache 2.0
@@ -385,7 +385,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
xml-apis 1.4.01: https://mvnrepository.com/artifact/xml-apis/xml-apis/1.4.01, Apache 2.0 and W3C
zookeeper 3.4.14: https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper/3.4.14, Apache 2.0
-
========================================================================
BSD licenses
========================================================================
diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml
index 798c971..a57d36a 100644
--- a/dolphinscheduler-plugin-api/pom.xml
+++ b/dolphinscheduler-plugin-api/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-plugin-api</artifactId>
<name>${project.artifactId}</name>
diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml
index 094b62c..855f2d2 100644
--- a/dolphinscheduler-remote/pom.xml
+++ b/dolphinscheduler-remote/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml
index 10fa58f..f00fbed 100644
--- a/dolphinscheduler-server/pom.xml
+++ b/dolphinscheduler-server/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>1.3.4-SNAPSHOT</version>
+ <version>1.3.5-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-server</artifactId>
<name>dolphinscheduler-server</name>
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
index 9543416..9c47fb9 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
@@ -16,11 +16,14 @@
*/
package org.apache.dolphinscheduler.server.log;
+import static org.apache.dolphinscheduler.common.utils.LoggerUtils.TASK_APPID_LOG_FORMAT;
+
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
/**
* task log filter
@@ -43,7 +46,9 @@ public class TaskLogFilter extends Filter<ILoggingEvent> {
*/
@Override
public FilterReply decide(ILoggingEvent event) {
- if (event.getThreadName().startsWith(LoggerUtils.TASK_LOGGER_THREAD_NAME) || event.getLevel().isGreaterOrEqual(level)) {
+ if (event.getThreadName().startsWith(LoggerUtils.TASK_LOGGER_THREAD_NAME)
+ || event.getLoggerName().startsWith(" - " + TASK_APPID_LOG_FORMAT)
+ || event.getLevel().isGreaterOrEqual(level)) {
return FilterReply.ACCEPT;
}
return FilterReply.DENY;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index 1a3bf6d..20d3ec1 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -19,10 +19,7 @@ package org.apache.dolphinscheduler.server.master.consumer;
import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.enums.ResourceType;
-import org.apache.dolphinscheduler.common.enums.TaskType;
-import org.apache.dolphinscheduler.common.enums.UdfType;
+import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
@@ -51,6 +48,7 @@ import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -69,7 +67,7 @@ public class TaskPriorityQueueConsumer extends Thread{
* taskUpdateQueue
*/
@Autowired
- private TaskPriorityQueue taskPriorityQueue;
+ private TaskPriorityQueue<String> taskPriorityQueue;
/**
* processService
@@ -116,8 +114,15 @@ public class TaskPriorityQueueConsumer extends Thread{
failedDispatchTasks.add(taskPriorityInfo);
}
}
- for(String dispatchFailedTask : failedDispatchTasks){
- taskPriorityQueue.put(dispatchFailedTask);
+ if (!failedDispatchTasks.isEmpty()) {
+ for (String dispatchFailedTask : failedDispatchTasks) {
+ taskPriorityQueue.put(dispatchFailedTask);
+ }
+ // If there are tasks in a cycle that cannot find the worker group,
+ // sleep for 1 second
+ if (taskPriorityQueue.size() <= failedDispatchTasks.size()) {
+ TimeUnit.MILLISECONDS.sleep(Constants.SLEEP_TIME_MILLIS);
+ }
}
}catch (Exception e){
logger.error("dispatcher task error",e);
@@ -132,7 +137,7 @@ public class TaskPriorityQueueConsumer extends Thread{
* @param taskInstanceId taskInstanceId
* @return result
*/
- private boolean dispatch(int taskInstanceId){
+ protected boolean dispatch(int taskInstanceId) {
boolean result = false;
try {
TaskExecutionContext context = getTaskExecutionContext(taskInstanceId);
@@ -253,8 +258,8 @@ public class TaskPriorityQueueConsumer extends Thread{
* @param dataxTaskExecutionContext dataxTaskExecutionContext
* @param taskNode taskNode
*/
- private void setDataxTaskRelation(DataxTaskExecutionContext dataxTaskExecutionContext, TaskNode taskNode) {
- DataxParameters dataxParameters = JSONObject.parseObject(taskNode.getParams(), DataxParameters.class);
+ protected void setDataxTaskRelation(DataxTaskExecutionContext dataxTaskExecutionContext, TaskNode taskNode) {
+ DataxParameters dataxParameters = JSONUtils.parseObject(taskNode.getParams(), DataxParameters.class);
DataSource dataSource = processService.findDataSourceById(dataxParameters.getDataSource());
DataSource dataTarget = processService.findDataSourceById(dataxParameters.getDataTarget());
@@ -275,29 +280,32 @@ public class TaskPriorityQueueConsumer extends Thread{
/**
- * set datax task relation
+ * set sqoop task relation
* @param sqoopTaskExecutionContext sqoopTaskExecutionContext
* @param taskNode taskNode
*/
private void setSqoopTaskRelation(SqoopTaskExecutionContext sqoopTaskExecutionContext, TaskNode taskNode) {
SqoopParameters sqoopParameters = JSONObject.parseObject(taskNode.getParams(), SqoopParameters.class);
- SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class);
- TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class);
+ // sqoop job type is template set task relation
+ if (sqoopParameters.getJobType().equals(SqoopJobType.TEMPLATE.getDescp())) {
+ SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class);
+ TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class);
- DataSource dataSource = processService.findDataSourceById(sourceMysqlParameter.getSrcDatasource());
- DataSource dataTarget = processService.findDataSourceById(targetMysqlParameter.getTargetDatasource());
+ DataSource dataSource = processService.findDataSourceById(sourceMysqlParameter.getSrcDatasource());
+ DataSource dataTarget = processService.findDataSourceById(targetMysqlParameter.getTargetDatasource());
- if (dataSource != null){
- sqoopTaskExecutionContext.setDataSourceId(dataSource.getId());
- sqoopTaskExecutionContext.setSourcetype(dataSource.getType().getCode());
- sqoopTaskExecutionContext.setSourceConnectionParams(dataSource.getConnectionParams());
- }
+ if (dataSource != null){
+ sqoopTaskExecutionContext.setDataSourceId(dataSource.getId());
+ sqoopTaskExecutionContext.setSourcetype(dataSource.getType().getCode());
+ sqoopTaskExecutionContext.setSourceConnectionParams(dataSource.getConnectionParams());
+ }
- if (dataTarget != null){
- sqoopTaskExecutionContext.setDataTargetId(dataTarget.getId());
- sqoopTaskExecutionContext.setTargetType(dataTarget.getType().getCode());
- sqoopTaskExecutionContext.setTargetConnectionParams(dataTarget.getConnectionParams());
+ if (dataTarget != null){
+ sqoopTaskExecutionContext.setDataTargetId(dataTarget.getId());
+ sqoopTaskExecutionContext.setTargetType(dataTarget.getType().getCode());
+ sqoopTaskExecutionContext.setTargetConnectionParams(dataTarget.getConnectionParams());
+ }
}
}
@@ -366,8 +374,8 @@ public class TaskPriorityQueueConsumer extends Thread{
/**
* get resource map key is full name and value is tenantCode
*/
- private Map<String,String> getResourceFullNames(TaskNode taskNode) {
- Map<String,String> resourceMap = new HashMap<>();
+ protected Map<String, String> getResourceFullNames(TaskNode taskNode) {
+ Map<String, String> resourceMap = new HashMap<>();
AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams());
if (baseParam != null) {
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java
index f365db7..d09ed71 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java
@@ -134,16 +134,17 @@ public class TaskResponseService {
case ACK:
try {
TaskInstance taskInstance = processService.findTaskInstanceById(taskResponseEvent.getTaskInstanceId());
- if (taskInstance != null){
- processService.changeTaskState(taskResponseEvent.getState(),
- taskResponseEvent.getStartTime(),
- taskResponseEvent.getWorkerAddress(),
- taskResponseEvent.getExecutePath(),
- taskResponseEvent.getLogPath(),
- taskResponseEvent.getTaskInstanceId());
+ if (taskInstance != null) {
+ ExecutionStatus status = taskInstance.getState().typeIsFinished() ? taskInstance.getState() : taskResponseEvent.getState();
+ processService.changeTaskState(status,
+ taskResponseEvent.getStartTime(),
+ taskResponseEvent.getWorkerAddress(),
+ taskResponseEvent.getExecutePath(),
+ taskResponseEvent.getLogPath(),
+ taskResponseEvent.getTaskInstanceId());
}
// if taskInstance is null (maybe deleted) . retry will be meaningless . so ack success
- DBTaskAckCommand taskAckCommand = new DBTaskAckCommand(ExecutionStatus.SUCCESS.getCode(),taskResponseEvent.getTaskInstanceId());
+ DBTaskAckCommand taskAckCommand = new DBTaskAckCommand(ExecutionStatus.SUCCESS.getCode(), taskResponseEvent.getTaskInstanceId());
channel.writeAndFlush(taskAckCommand.convert2Command());
}catch (Exception e){
logger.error("worker ack master error",e);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
index e8c9833..3bc2cb4 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
@@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.master.runner;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.graph.DAG;
@@ -38,12 +37,10 @@ import org.apache.dolphinscheduler.server.utils.AlertManager;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
-
+import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@@ -55,7 +52,6 @@ import static org.apache.dolphinscheduler.common.Constants.*;
* master exec thread,split dag
*/
public class MasterExecThread implements Runnable {
-
/**
* logger of MasterExecThread
*/
@@ -97,9 +93,9 @@ public class MasterExecThread implements Runnable {
private Map<String, TaskInstance> completeTaskList = new ConcurrentHashMap<>();
/**
- * ready to submit task list
+ * ready to submit task queue
*/
- private Map<String, TaskInstance> readyToSubmitTaskList = new ConcurrentHashMap<>();
+ private PeerTaskInstancePriorityQueue readyToSubmitTaskQueue = new PeerTaskInstancePriorityQueue();
/**
* depend failed task map
@@ -192,8 +188,6 @@ public class MasterExecThread implements Runnable {
processService.updateProcessInstance(processInstance);
}finally {
taskExecService.shutdown();
- // post handle
- postHandle();
}
}
@@ -382,27 +376,6 @@ public class MasterExecThread implements Runnable {
}
/**
- * process post handle
- */
- private void postHandle() {
- logger.info("develop mode is: {}", CommonUtils.isDevelopMode());
-
- if (!CommonUtils.isDevelopMode()) {
- // get exec dir
- String execLocalPath = org.apache.dolphinscheduler.common.utils.FileUtils
- .getProcessExecDir(processInstance.getProcessDefinition().getProjectId(),
- processInstance.getProcessDefinitionId(),
- processInstance.getId());
-
- try {
- FileUtils.deleteDirectory(new File(execLocalPath));
- } catch (IOException e) {
- logger.error("delete exec dir failed ", e);
- }
- }
- }
-
- /**
* submit task to execute
* @param taskInstance task instance
* @return TaskInstance
@@ -514,7 +487,7 @@ public class MasterExecThread implements Runnable {
}
// if previous node success , post node submit
for(TaskInstance task : taskInstances){
- if(readyToSubmitTaskList.containsKey(task.getName())){
+ if(readyToSubmitTaskQueue.contains(task)){
continue;
}
if(completeTaskList.containsKey(task.getName())){
@@ -648,7 +621,7 @@ public class MasterExecThread implements Runnable {
return true;
}
if (processInstance.getFailureStrategy() == FailureStrategy.CONTINUE) {
- return readyToSubmitTaskList.size() == 0 || activeTaskNode.size() == 0;
+ return readyToSubmitTaskQueue.size() == 0 || activeTaskNode.size() == 0;
}
}
return false;
@@ -678,7 +651,7 @@ public class MasterExecThread implements Runnable {
List<TaskInstance> pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE);
if(CollectionUtils.isNotEmpty(pauseList)
|| !isComplementEnd()
- || readyToSubmitTaskList.size() > 0){
+ || readyToSubmitTaskQueue.size() > 0){
return ExecutionStatus.PAUSE;
}else{
return ExecutionStatus.SUCCESS;
@@ -729,7 +702,7 @@ public class MasterExecThread implements Runnable {
// success
if(state == ExecutionStatus.RUNNING_EXEUTION){
List<TaskInstance> killTasks = getCompleteTaskByState(ExecutionStatus.KILL);
- if(readyToSubmitTaskList.size() > 0){
+ if(readyToSubmitTaskQueue.size() > 0){
//tasks currently pending submission, no retries, indicating that depend is waiting to complete
return ExecutionStatus.RUNNING_EXEUTION;
}else if(CollectionUtils.isNotEmpty(killTasks)){
@@ -752,8 +725,8 @@ public class MasterExecThread implements Runnable {
boolean result = false;
- for(String taskName : readyToSubmitTaskList.keySet()){
- TaskInstance task = readyToSubmitTaskList.get(taskName);
+ for (Iterator<TaskInstance> iter = readyToSubmitTaskQueue.iterator(); iter.hasNext();) {
+ TaskInstance task = iter.next();
if(task.getState().typeIsFailure()){
result = true;
break;
@@ -817,7 +790,11 @@ public class MasterExecThread implements Runnable {
*/
private void addTaskToStandByList(TaskInstance taskInstance){
logger.info("add task to stand by list: {}", taskInstance.getName());
- readyToSubmitTaskList.putIfAbsent(taskInstance.getName(), taskInstance);
+ try {
+ readyToSubmitTaskQueue.put(taskInstance);
+ } catch (Exception e) {
+ logger.error("add task instance to readyToSubmitTaskQueue error");
+ }
}
/**
@@ -826,7 +803,11 @@ public class MasterExecThread implements Runnable {
*/
private void removeTaskFromStandbyList(TaskInstance taskInstance){
logger.info("remove task from stand by list: {}", taskInstance.getName());
- readyToSubmitTaskList.remove(taskInstance.getName());
+ try {
+ readyToSubmitTaskQueue.remove(taskInstance);
+ } catch (Exception e) {
+ logger.error("remove task instance from readyToSubmitTaskQueue error");
+ }
}
/**
@@ -834,10 +815,11 @@ public class MasterExecThread implements Runnable {
* @return Boolean whether has retry task in standby
*/
private boolean hasRetryTaskInStandBy(){
- for (Map.Entry<String, TaskInstance> entry: readyToSubmitTaskList.entrySet()) {
- if(entry.getValue().getState().typeIsFailure()){
+ for (Iterator<TaskInstance> iter = readyToSubmitTaskQueue.iterator(); iter.hasNext();) {
+ if(iter.next().getState().typeIsFailure()){
return true;
}
+
}
return false;
}
@@ -1019,20 +1001,26 @@ public class MasterExecThread implements Runnable {
* handling the list of tasks to be submitted
*/
private void submitStandByTask(){
- for(Map.Entry<String, TaskInstance> entry: readyToSubmitTaskList.entrySet()) {
- TaskInstance task = entry.getValue();
- DependResult dependResult = getDependResultForTask(task);
- if(DependResult.SUCCESS == dependResult){
- if(retryTaskIntervalOverTime(task)){
- submitTaskExec(task);
+
+ try {
+ int length = readyToSubmitTaskQueue.size();
+ for (int i=0;i<length;i++) {
+ TaskInstance task = readyToSubmitTaskQueue.peek();
+ DependResult dependResult = getDependResultForTask(task);
+ if(DependResult.SUCCESS == dependResult){
+ if(retryTaskIntervalOverTime(task)){
+ submitTaskExec(task);
+ removeTaskFromStandbyList(task);
+ }
+ }else if(DependResult.FAILED == dependResult){
+ // if the dependency fails, the current node is not submitted and the state changes to failure.
+ dependFailedTask.put(task.getName(), task);
removeTaskFromStandbyList(task);
+ logger.info("task {},id:{} depend result : {}",task.getName(), task.getId(), dependResult);
}
- }else if(DependResult.FAILED == dependResult){
- // if the dependency fails, the current node is not submitted and the state changes to failure.
- dependFailedTask.put(entry.getKey(), task);
- removeTaskFromStandbyList(task);
- logger.info("task {},id:{} depend result : {}",task.getName(), task.getId(), dependResult);
}
+ } catch (Exception e) {
+ logger.error("submit standby task error",e);
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
index 5074a5e..270f43e 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
@@ -16,392 +16,436 @@
*/
package org.apache.dolphinscheduler.server.utils;
-import java.nio.charset.StandardCharsets;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
-import org.apache.commons.io.FileUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.service.log.LogClientService;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
/**
- * mainly used to get the start command line of a process
+ * mainly used to get the start command line of a process.
*/
public class ProcessUtils {
- /**
- * logger
- */
- private final static Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
-
- /**
- * build command line characters
- * @param commandList command list
- * @return command
- * @throws IOException io exception
- */
- public static String buildCommandStr(List<String> commandList) throws IOException {
- String cmdstr;
- String[] cmd = commandList.toArray(new String[commandList.size()]);
- SecurityManager security = System.getSecurityManager();
- boolean allowAmbiguousCommands = false;
- if (security == null) {
- allowAmbiguousCommands = true;
- String value = System.getProperty("jdk.lang.Process.allowAmbiguousCommands");
- if (value != null) {
- allowAmbiguousCommands = !"false".equalsIgnoreCase(value);
- }
+ /**
+ * logger
+ */
+ private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
+
+ /**
+ * Initialization regularization, solve the problem of pre-compilation performance,
+ * avoid the thread safety problem of multi-thread operation.
+ */
+ private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)");
+
+ private static final Pattern LINUXPATTERN = Pattern.compile("(\\d+)");
+
+ private static final String LOCAL_PROCESS_EXEC = "jdk.lang.Process.allowAmbiguousCommands";
+
+ /**
+ * build command line characters.
+ *
+ * @param commandList command list
+ * @return command
+ * @throws IOException io exception
+ */
+ public static String buildCommandStr(List<String> commandList) throws IOException {
+ String cmdstr;
+ String[] cmd = commandList.toArray(new String[0]);
+ SecurityManager security = System.getSecurityManager();
+ boolean allowAmbiguousCommands = isAllowAmbiguousCommands(security);
+ if (allowAmbiguousCommands) {
+
+ String executablePath = new File(cmd[0]).getPath();
+
+ if (needsEscaping(VERIFICATION_LEGACY, executablePath)) {
+ executablePath = quoteString(executablePath);
+ }
+
+ cmdstr = createCommandLine(
+ VERIFICATION_LEGACY, executablePath, cmd);
+ } else {
+ String executablePath;
+ try {
+ executablePath = getExecutablePath(cmd[0]);
+ } catch (IllegalArgumentException e) {
+
+ StringBuilder join = new StringBuilder();
+ for (String s : cmd) {
+ join.append(s).append(' ');
+ }
+
+ cmd = getTokensFromCommand(join.toString());
+ executablePath = getExecutablePath(cmd[0]);
+
+ // Check new executable name once more
+ if (security != null) {
+ security.checkExec(executablePath);
+ }
+ }
+
+ cmdstr = createCommandLine(
+
+ isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd);
+ }
+ return cmdstr;
}
- if (allowAmbiguousCommands) {
- String executablePath = new File(cmd[0]).getPath();
-
- if (needsEscaping(VERIFICATION_LEGACY, executablePath)) {
- executablePath = quoteString(executablePath);
- }
-
- cmdstr = createCommandLine(
- VERIFICATION_LEGACY, executablePath, cmd);
- } else {
- String executablePath;
- try {
- executablePath = getExecutablePath(cmd[0]);
- } catch (IllegalArgumentException e) {
-
- StringBuilder join = new StringBuilder();
- for (String s : cmd) {
- join.append(s).append(' ');
+ /**
+ * check is allow ambiguous commands
+ *
+ * @param security security manager
+ * @return allow ambiguous command flag
+ */
+ private static boolean isAllowAmbiguousCommands(SecurityManager security) {
+ boolean allowAmbiguousCommands = false;
+ if (security == null) {
+ allowAmbiguousCommands = true;
+ String value = System.getProperty(LOCAL_PROCESS_EXEC);
+ if (value != null) {
+ allowAmbiguousCommands = !Constants.STRING_FALSE.equalsIgnoreCase(value);
+ }
}
+ return allowAmbiguousCommands;
+ }
- cmd = getTokensFromCommand(join.toString());
- executablePath = getExecutablePath(cmd[0]);
-
- // Check new executable name once more
- if (security != null) {
- security.checkExec(executablePath);
- }
- }
+ /**
+ * get executable path.
+ *
+ * @param path path
+ * @return executable path
+ * @throws IOException io exception
+ */
+ private static String getExecutablePath(String path) throws IOException {
+ boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
+
+ File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
+ return fileToRun.getPath();
+ }
+ /**
+ * whether is shell file.
+ *
+ * @param executablePath executable path
+ * @return true if endsWith .CMD or .BAT
+ */
+ private static boolean isShellFile(String executablePath) {
+ String upPath = executablePath.toUpperCase();
+ return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
+ }
- cmdstr = createCommandLine(
+ /**
+ * quote string
+ *
+ * @param arg argument
+ * @return format arg
+ */
+ private static String quoteString(String arg) {
+ StringBuilder argbuf = new StringBuilder(arg.length() + 2);
+ return argbuf.append('"').append(arg).append('"').toString();
+ }
- isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd);
+ /**
+ * get tokens from command.
+ *
+ * @param command command
+ * @return token string array
+ */
+ private static String[] getTokensFromCommand(String command) {
+ ArrayList<String> matchList = new ArrayList<>(8);
+ Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
+ while (regexMatcher.find()) {
+ matchList.add(regexMatcher.group());
+ }
+ return matchList.toArray(new String[0]);
}
- return cmdstr;
- }
-
- /**
- * get executable path
- *
- * @param path path
- * @return executable path
- * @throws IOException io exception
- */
- private static String getExecutablePath(String path) throws IOException {
- boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
-
- File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
- return fileToRun.getPath();
- }
-
- /**
- * whether is shell file
- *
- * @param executablePath executable path
- * @return true if endsWith .CMD or .BAT
- */
- private static boolean isShellFile(String executablePath) {
- String upPath = executablePath.toUpperCase();
- return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
- }
-
- /**
- * quote string
- *
- * @param arg argument
- * @return format arg
- */
- private static String quoteString(String arg) {
- StringBuilder argbuf = new StringBuilder(arg.length() + 2);
- return argbuf.append('"').append(arg).append('"').toString();
- }
-
- /**
- * get tokens from command
- *
- * @param command command
- * @return token string array
- */
- private static String[] getTokensFromCommand(String command) {
- ArrayList<String> matchList = new ArrayList<>(8);
- Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
- while (regexMatcher.find()) {
- matchList.add(regexMatcher.group());
+
+ /**
+ * Lazy Pattern
+ */
+ private static class LazyPattern {
+ /**
+ * Escape-support version:
+ * "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
+ */
+ private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
}
- return matchList.toArray(new String[matchList.size()]);
- }
-
- /**
- * Lazy Pattern
- */
- private static class LazyPattern {
- // Escape-support version:
- // "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
- private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
- }
-
- /**
- * verification cmd bat
- */
- private static final int VERIFICATION_CMD_BAT = 0;
-
- /**
- * verification win32
- */
- private static final int VERIFICATION_WIN32 = 1;
-
- /**
- * verification legacy
- */
- private static final int VERIFICATION_LEGACY = 2;
-
- /**
- * escape verification
- */
- private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
-
- {' ', '\t', '<', '>'}, {' ', '\t'}};
-
- /**
- * matcher
- */
- private static Matcher matcher;
-
- /**
- * create command line
- * @param verificationType verification type
- * @param executablePath executable path
- * @param cmd cmd
- * @return command line
- */
- private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
- StringBuilder cmdbuf = new StringBuilder(80);
-
- cmdbuf.append(executablePath);
-
- for (int i = 1; i < cmd.length; ++i) {
- cmdbuf.append(' ');
- String s = cmd[i];
- if (needsEscaping(verificationType, s)) {
- cmdbuf.append('"').append(s);
-
- if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
- cmdbuf.append('\\');
+
+ /**
+ * verification cmd bat
+ */
+ private static final int VERIFICATION_CMD_BAT = 0;
+
+ /**
+ * verification win32
+ */
+ private static final int VERIFICATION_WIN32 = 1;
+
+ /**
+ * verification legacy
+ */
+ private static final int VERIFICATION_LEGACY = 2;
+
+ /**
+ * escape verification
+ */
+ private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
+
+ {' ', '\t', '<', '>'}, {' ', '\t'}};
+
+ /**
+ * create command line
+ *
+ * @param verificationType verification type
+ * @param executablePath executable path
+ * @param cmd cmd
+ * @return command line
+ */
+ private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
+ StringBuilder cmdbuf = new StringBuilder(80);
+
+ cmdbuf.append(executablePath);
+
+ for (int i = 1; i < cmd.length; ++i) {
+ cmdbuf.append(' ');
+ String s = cmd[i];
+ if (needsEscaping(verificationType, s)) {
+ cmdbuf.append('"').append(s);
+
+ if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
+ cmdbuf.append('\\');
+ }
+ cmdbuf.append('"');
+ } else {
+ cmdbuf.append(s);
+ }
}
- cmdbuf.append('"');
- } else {
- cmdbuf.append(s);
- }
+ return cmdbuf.toString();
}
- return cmdbuf.toString();
- }
-
- /**
- * whether is quoted
- * @param noQuotesInside
- * @param arg
- * @param errorMessage
- * @return boolean
- */
- private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) {
- int lastPos = arg.length() - 1;
- if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
- // The argument has already been quoted.
- if (noQuotesInside) {
- if (arg.indexOf('"', 1) != lastPos) {
- // There is ["] inside.
- throw new IllegalArgumentException(errorMessage);
+
+ /**
+ * whether is quoted
+ *
+ * @param noQuotesInside no quotes inside
+ * @param arg arg
+ * @param errorMessage error message
+ * @return boolean
+ */
+ private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) {
+ int lastPos = arg.length() - 1;
+ if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
+ // The argument has already been quoted.
+ if (noQuotesInside && arg.indexOf('"', 1) != lastPos) {
+ // There is ["] inside.
+ throw new IllegalArgumentException(errorMessage);
+ }
+ return true;
}
- }
- return true;
- }
- if (noQuotesInside) {
- if (arg.indexOf('"') >= 0) {
- // There is ["] inside.
- throw new IllegalArgumentException(errorMessage);
- }
- }
- return false;
- }
-
- /**
- * whether needs escaping
- *
- * @param verificationType verification type
- * @param arg arg
- * @return boolean
- */
- private static boolean needsEscaping(int verificationType, String arg) {
-
- boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
-
- if (!argIsQuoted) {
- char[] testEscape = ESCAPE_VERIFICATION[verificationType];
- for (int i = 0; i < testEscape.length; ++i) {
- if (arg.indexOf(testEscape[i]) >= 0) {
- return true;
+ if (noQuotesInside && arg.indexOf('"') >= 0) {
+ // There is ["] inside.
+ throw new IllegalArgumentException(errorMessage);
}
- }
+ return false;
}
- return false;
- }
-
- /**
- * kill yarn application
- *
- * @param appIds app id list
- * @param logger logger
- * @param tenantCode tenant code
- * @param executePath execute path
- * @throws IOException io exception
- */
- public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode,String executePath)
- throws IOException {
- if (appIds.size() > 0) {
- String appid = appIds.get(appIds.size() - 1);
- String commandFile = String
- .format("%s/%s.kill", executePath, appid);
- String cmd = "yarn application -kill " + appid;
- try {
- StringBuilder sb = new StringBuilder();
- sb.append("#!/bin/sh\n");
- sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
- sb.append("cd $BASEDIR\n");
- if (CommonUtils.getSystemEnvPath() != null) {
- sb.append("source " + CommonUtils.getSystemEnvPath() + "\n");
- }
- sb.append("\n\n");
- sb.append(cmd);
-
- File f = new File(commandFile);
- if (!f.exists()) {
- FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
+ /**
+ * whether needs escaping
+ *
+ * @param verificationType verification type
+ * @param arg arg
+ * @return boolean
+ */
+ private static boolean needsEscaping(int verificationType, String arg) {
+
+ boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
+
+ if (!argIsQuoted) {
+ char[] testEscape = ESCAPE_VERIFICATION[verificationType];
+ for (char c : testEscape) {
+ if (arg.indexOf(c) >= 0) {
+ return true;
+ }
+ }
}
+ return false;
+ }
- String runCmd = "sh " + commandFile;
- if (StringUtils.isNotEmpty(tenantCode)) {
- runCmd = "sudo -u " + tenantCode + " " + runCmd;
+ /**
+ * kill yarn application
+ *
+ * @param appIds app id list
+ * @param logger logger
+ * @param tenantCode tenant code
+ * @param executePath execute path
+ */
+ public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) {
+ if (CollectionUtils.isNotEmpty(appIds)) {
+
+ for (String appId : appIds) {
+ try {
+ ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId);
+
+ if (!applicationStatus.typeIsFinished()) {
+ String commandFile = String
+ .format("%s/%s.kill", executePath, appId);
+ String cmd = "yarn application -kill " + appId;
+ execYarnKillCommand(logger, tenantCode, appId, commandFile, cmd);
+ }
+ } catch (Exception e) {
+ logger.error(String.format("Get yarn application app id [%s] status failed: [%s]", appId, e.getMessage()));
+ }
+ }
}
+ }
- logger.info("kill cmd:{}", runCmd);
-
- Runtime.getRuntime().exec(runCmd);
- } catch (Exception e) {
- logger.error("kill application error", e);
- }
+ /**
+ * build kill command for yarn application
+ *
+ * @param logger logger
+ * @param tenantCode tenant code
+ * @param appId app id
+ * @param commandFile command file
+ * @param cmd cmd
+ */
+ private static void execYarnKillCommand(Logger logger, String tenantCode, String appId, String commandFile, String cmd) {
+ try {
+ StringBuilder sb = new StringBuilder();
+ sb.append("#!/bin/sh\n");
+ sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
+ sb.append("cd $BASEDIR\n");
+ if (CommonUtils.getSystemEnvPath() != null) {
+ sb.append("source ").append(CommonUtils.getSystemEnvPath()).append("\n");
+ }
+ sb.append("\n\n");
+ sb.append(cmd);
+
+ File f = new File(commandFile);
+
+ if (!f.exists()) {
+ FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
+ }
+
+ String runCmd = String.format("%s %s", Constants.SH, commandFile);
+ if (StringUtils.isNotEmpty(tenantCode)) {
+ runCmd = "sudo -u " + tenantCode + " " + runCmd;
+ }
+
+ logger.info("kill cmd:{}", runCmd);
+ OSUtils.exeCmd(runCmd);
+ } catch (Exception e) {
+ logger.error(String.format("Kill yarn application app id [%s] failed: [%s]", appId, e.getMessage()));
+ }
}
- }
- /**
- * kill tasks according to different task types
- *
- * @param taskExecutionContext taskExecutionContext
- */
- public static void kill(TaskExecutionContext taskExecutionContext) {
- try {
- int processId = taskExecutionContext.getProcessId();
- if(processId == 0 ){
- logger.error("process kill failed, process id :{}, task id:{}",
- processId, taskExecutionContext.getTaskInstanceId());
- return ;
- }
+ /**
+ * kill tasks according to different task types
+ *
+ * @param taskExecutionContext taskExecutionContext
+ */
+ public static void kill(TaskExecutionContext taskExecutionContext) {
+ try {
+ int processId = taskExecutionContext.getProcessId();
+ if (processId == 0) {
+ logger.error("process kill failed, process id :{}, task id:{}",
+ processId, taskExecutionContext.getTaskInstanceId());
+ return;
+ }
- String cmd = String.format("sudo kill -9 %s", getPidsStr(processId));
+ String cmd = String.format("sudo kill -9 %s", getPidsStr(processId));
- logger.info("process id:{}, cmd:{}", processId, cmd);
+ logger.info("process id:{}, cmd:{}", processId, cmd);
- OSUtils.exeCmd(cmd);
+ OSUtils.exeCmd(cmd);
- // find log and kill yarn job
- killYarnJob(taskExecutionContext);
+ // find log and kill yarn job
+ killYarnJob(taskExecutionContext);
- } catch (Exception e) {
- logger.error("kill task failed", e);
- }
- }
-
- /**
- * get pids str
- *
- * @param processId process id
- * @return pids
- * @throws Exception exception
- */
- public static String getPidsStr(int processId)throws Exception{
- StringBuilder sb = new StringBuilder();
- Matcher mat;
- // pstree pid get sub pids
- if (OSUtils.isMacOS()) {
- String pids = OSUtils.exeCmd("pstree -sp " + processId);
- mat = Pattern.compile("-[+|-]-\\s(\\d+)").matcher(pids);
- } else {
- String pids = OSUtils.exeCmd("pstree -p " + processId);
- mat = Pattern.compile("(\\d+)").matcher(pids);
+ } catch (Exception e) {
+ logger.error("kill task failed", e);
+ }
}
- while (mat.find()){
- sb.append(mat.group(1)).append(" ");
- }
- return sb.toString().trim();
- }
-
- /**
- * find logs and kill yarn tasks
- *
- * @param taskExecutionContext taskExecutionContext
- */
- public static void killYarnJob(TaskExecutionContext taskExecutionContext) {
- try {
- Thread.sleep(Constants.SLEEP_TIME_MILLIS);
- LogClientService logClient = null;
- String log = null;
- try {
- logClient = new LogClientService();
- log = logClient.viewLog(Host.of(taskExecutionContext.getHost()).getIp(),
- Constants.RPC_PORT,
- taskExecutionContext.getLogPath());
- } finally {
- if(logClient != null){
- logClient.close();
- }
- }
- if (StringUtils.isNotEmpty(log)) {
- List<String> appIds = LoggerUtils.getAppIds(log, logger);
- String workerDir = taskExecutionContext.getExecutePath();
- if (StringUtils.isEmpty(workerDir)) {
- logger.error("task instance work dir is empty");
- throw new RuntimeException("task instance work dir is empty");
+ /**
+ * get pids str
+ *
+ * @param processId process id
+ * @return pids pid String
+ * @throws Exception exception
+ */
+ public static String getPidsStr(int processId) throws Exception {
+ StringBuilder sb = new StringBuilder();
+ Matcher mat = null;
+ // pstree pid get sub pids
+ if (OSUtils.isMacOS()) {
+ String pids = OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, processId));
+ if (null != pids) {
+ mat = MACPATTERN.matcher(pids);
+ }
+ } else {
+ String pids = OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, processId));
+ mat = LINUXPATTERN.matcher(pids);
}
- if (appIds.size() > 0) {
- cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath());
+
+ if (null != mat) {
+ while (mat.find()) {
+ sb.append(mat.group(1)).append(" ");
+ }
}
- }
- } catch (Exception e) {
- logger.error("kill yarn job failure",e);
+ return sb.toString().trim();
+ }
+
+ /**
+ * find logs and kill yarn tasks
+ *
+ * @param taskExecutionContext taskExecutionContext
+ */
+ public static void killYarnJob(TaskExecutionContext taskExecutionContext) {
+ try {
+ Thread.sleep(Constants.SLEEP_TIME_MILLIS);
+ LogClientService logClient = null;
+ String log;
+ try {
+ logClient = new LogClientService();
+ log = logClient.viewLog(Host.of(taskExecutionContext.getHost()).getIp(),
+ Constants.RPC_PORT,
+ taskExecutionContext.getLogPath());
+ } finally {
+ if (logClient != null) {
+ logClient.close();
+ }
+ }
+ if (StringUtils.isNotEmpty(log)) {
+ List<String> appIds = LoggerUtils.getAppIds(log, logger);
+ String workerDir = taskExecutionContext.getExecutePath();
+ if (StringUtils.isEmpty(workerDir)) {
+ logger.error("task instance work dir is empty");
+ throw new RuntimeException("task instance work dir is empty");
+ }
+ if (CollectionUtils.isNotEmpty(appIds)) {
+ cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath());
+ }
+ }
+
+ } catch (Exception e) {
+ logger.error("kill yarn job failure", e);
+ }
}
- }
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/TaskExecutionContextCacheManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/TaskExecutionContextCacheManager.java
index 7df8e01..c663365 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/TaskExecutionContextCacheManager.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/TaskExecutionContextCacheManager.java
@@ -42,7 +42,16 @@ public interface TaskExecutionContextCacheManager {
/**
* remove taskInstance by taskInstanceId
+ *
* @param taskInstanceId taskInstanceId
*/
void removeByTaskInstanceId(Integer taskInstanceId);
+
+ /**
+ * If the value for the specified key is present and non-null,then perform the update，otherwise it will return false
+ *
+ * @param taskExecutionContext taskExecutionContext
+ * @return status
+ */
+ boolean updateTaskExecutionContext(TaskExecutionContext taskExecutionContext);
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java
index 009332f..418c68d 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java
@@ -59,10 +59,17 @@ public class TaskExecutionContextCacheManagerImpl implements TaskExecutionContex
/**
* remove taskInstance by taskInstanceId
+ *
* @param taskInstanceId taskInstanceId
*/
@Override
public void removeByTaskInstanceId(Integer taskInstanceId) {
taskExecutionContextCache.remove(taskInstanceId);
}
+
+ @Override
+ public boolean updateTaskExecutionContext(TaskExecutionContext taskExecutionContext) {
+ taskExecutionContextCache.computeIfPresent(taskExecutionContext.getTaskInstanceId(), (k, v) -> taskExecutionContext);
+ return taskExecutionContextCache.containsKey(taskExecutionContext.getTaskInstanceId());
+ }
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
index cbd8212..f8b9567 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
@@ -25,10 +25,10 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Event;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskType;
-import org.apache.dolphinscheduler.common.utils.OSUtils;
-import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -37,14 +37,19 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.server.worker.cache.ResponceCache;
+import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager;
+import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Date;
+import java.util.Optional;
import java.util.concurrent.ExecutorService;
/**
@@ -54,41 +59,73 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(TaskExecuteProcessor.class);
-
/**
* thread executor service
*/
private final ExecutorService workerExecService;
/**
- * worker config
+ * worker config
*/
private final WorkerConfig workerConfig;
/**
- * task callback service
+ * task callback service
*/
private final TaskCallbackService taskCallbackService;
- public TaskExecuteProcessor(){
+ /**
+ * taskExecutionContextCacheManager
+ */
+ private TaskExecutionContextCacheManager taskExecutionContextCacheManager;
+
+ public TaskExecuteProcessor() {
this.taskCallbackService = SpringApplicationContext.getBean(TaskCallbackService.class);
this.workerConfig = SpringApplicationContext.getBean(WorkerConfig.class);
this.workerExecService = ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", workerConfig.getWorkerExecThreads());
+ this.taskExecutionContextCacheManager = SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class);
+ }
+
+ /**
+ * Pre-cache task to avoid extreme situations when kill task. There is no such task in the cache
+ *
+ * @param taskExecutionContext task
+ */
+ private void setTaskCache(TaskExecutionContext taskExecutionContext) {
+ TaskExecutionContext preTaskCache = new TaskExecutionContext();
+ preTaskCache.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+ taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
}
@Override
public void process(Channel channel, Command command) {
Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(),
- String.format("invalid command type : %s", command.getType()));
+ String.format("invalid command type : %s", command.getType()));
TaskExecuteRequestCommand taskRequestCommand = FastJsonSerializer.deserialize(
- command.getBody(), TaskExecuteRequestCommand.class);
+ command.getBody(), TaskExecuteRequestCommand.class);
logger.info("received command : {}", taskRequestCommand);
- String contextJson = taskRequestCommand.getTaskExecutionContext();
+ if (taskRequestCommand == null) {
+ logger.error("task execute request command is null");
+ return;
+ }
+ String contextJson = taskRequestCommand.getTaskExecutionContext();
TaskExecutionContext taskExecutionContext = JSONObject.parseObject(contextJson, TaskExecutionContext.class);
+
+ if (taskExecutionContext == null) {
+ logger.error("task execution context is null");
+ return;
+ }
+ setTaskCache(taskExecutionContext);
+ // custom logger
+ Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
+ taskExecutionContext.getProcessDefineId(),
+ taskExecutionContext.getProcessInstanceId(),
+ taskExecutionContext.getTaskInstanceId()));
+
taskExecutionContext.setHost(OSUtils.getHost() + ":" + workerConfig.getListenPort());
taskExecutionContext.setStartTime(new Date());
taskExecutionContext.setLogPath(getTaskLogPath(taskExecutionContext));
@@ -97,18 +134,24 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
String execLocalPath = getExecLocalPath(taskExecutionContext);
logger.info("task instance local execute path : {} ", execLocalPath);
+ FileUtils.taskLoggerThreadLocal.set(taskLogger);
try {
FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, taskExecutionContext.getTenantCode());
- } catch (Exception ex){
- logger.error(String.format("create execLocalPath : %s", execLocalPath), ex);
+ } catch (Throwable ex) {
+ String errorLog = String.format("create execLocalPath : %s", execLocalPath);
+ LoggerUtils.logError(Optional.ofNullable(logger), errorLog, ex);
+ LoggerUtils.logError(Optional.ofNullable(taskLogger), errorLog, ex);
+ taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
}
+ FileUtils.taskLoggerThreadLocal.remove();
+
taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
new NettyRemoteChannel(channel, command.getOpaque()));
this.doAck(taskExecutionContext);
// submit task
- workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService));
+ workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger));
}
private void doAck(TaskExecutionContext taskExecutionContext){
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
index cf0c051..c9f2cd4 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
@@ -102,15 +102,17 @@ public class TaskKillProcessor implements NettyRequestProcessor {
* @return kill result
*/
private Pair<Boolean, List<String>> doKill(TaskKillRequestCommand killCommand){
- List<String> appIds = Collections.EMPTY_LIST;
+ List<String> appIds = Collections.emptyList();
try {
- TaskExecutionContext taskExecutionContext = taskExecutionContextCacheManager.getByTaskInstanceId(killCommand.getTaskInstanceId());
+ int taskInstanceId = killCommand.getTaskInstanceId();
+ TaskExecutionContext taskExecutionContext = taskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId);
Integer processId = taskExecutionContext.getProcessId();
- if (processId == null || processId.equals(0)){
- logger.error("process kill failed, process id :{}, task id:{}", processId, killCommand.getTaskInstanceId());
- return Pair.of(false, appIds);
+ if (processId.equals(0)) {
+ taskExecutionContextCacheManager.removeByTaskInstanceId(taskInstanceId);
+ logger.info("the task has not been executed and has been cancelled, task id:{}", taskInstanceId);
+ return Pair.of(true, appIds);
}
String cmd = String.format("sudo kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId()));
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
index fa3e6d2..81f14e7 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@@ -14,8 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.worker.runner;
+package org.apache.dolphinscheduler.server.worker.runner;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections.MapUtils;
@@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.worker.cache.ResponceCache;
@@ -36,18 +36,21 @@ import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskManager;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.*;
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
-import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * task scheduler thread
+ * task scheduler thread
*/
public class TaskExecuteThread implements Runnable {
@@ -57,17 +60,17 @@ public class TaskExecuteThread implements Runnable {
private final Logger logger = LoggerFactory.getLogger(TaskExecuteThread.class);
/**
- * task instance
+ * task instance
*/
private TaskExecutionContext taskExecutionContext;
/**
- * abstract task
+ * abstract task
*/
private AbstractTask task;
/**
- * task callback service
+ * task callback service
*/
private TaskCallbackService taskCallbackService;
@@ -77,14 +80,22 @@ public class TaskExecuteThread implements Runnable {
private TaskExecutionContextCacheManager taskExecutionContextCacheManager;
/**
+ * task logger
+ */
+ private Logger taskLogger;
+
+ /**
* constructor
* @param taskExecutionContext taskExecutionContext
* @param taskCallbackService taskCallbackService
*/
- public TaskExecuteThread(TaskExecutionContext taskExecutionContext, TaskCallbackService taskCallbackService){
+ public TaskExecuteThread(TaskExecutionContext taskExecutionContext
+ , TaskCallbackService taskCallbackService
+ , Logger taskLogger) {
this.taskExecutionContext = taskExecutionContext;
this.taskCallbackService = taskCallbackService;
this.taskExecutionContextCacheManager = SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class);
+ this.taskLogger = taskLogger;
}
@Override
@@ -113,14 +124,7 @@ public class TaskExecuteThread implements Runnable {
taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTaskInstanceId()));
- // custom logger
- Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
- taskExecutionContext.getProcessDefineId(),
- taskExecutionContext.getProcessInstanceId(),
- taskExecutionContext.getTaskInstanceId()));
-
- task = TaskManager.newTask(taskExecutionContext,
- taskLogger);
+ task = TaskManager.newTask(taskExecutionContext, taskLogger);
// task init
task.init();
@@ -135,7 +139,7 @@ public class TaskExecuteThread implements Runnable {
responseCommand.setProcessId(task.getProcessId());
responseCommand.setAppIds(task.getAppIds());
logger.info("task instance id : {},task final status : {}", taskExecutionContext.getTaskInstanceId(), task.getExitStatus());
- }catch (Exception e){
+ } catch (Exception e) {
logger.error("task scheduler failure", e);
kill();
responseCommand.setStatus(ExecutionStatus.FAILURE.getCode());
@@ -144,18 +148,47 @@ public class TaskExecuteThread implements Runnable {
responseCommand.setAppIds(task.getAppIds());
} finally {
taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
- ResponceCache.get().cache(taskExecutionContext.getTaskInstanceId(),responseCommand.convert2Command(),Event.RESULT);
+ ResponceCache.get().cache(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command(), Event.RESULT);
taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command());
+ clearTaskExecPath();
+ }
+ }
+ /**
+ * when task finish, clear execute path.
+ */
+ private void clearTaskExecPath() {
+ logger.info("develop mode is: {}", CommonUtils.isDevelopMode());
+
+ if (!CommonUtils.isDevelopMode()) {
+ // get exec dir
+ String execLocalPath = taskExecutionContext.getExecutePath();
+
+ if (StringUtils.isEmpty(execLocalPath)) {
+ logger.warn("task: {} exec local path is empty.", taskExecutionContext.getTaskName());
+ return;
+ }
+
+ if ("/".equals(execLocalPath)) {
+ logger.warn("task: {} exec local path is '/'， direct deletion is not allowed", taskExecutionContext.getTaskName());
+ return;
+ }
+
+ try {
+ org.apache.commons.io.FileUtils.deleteDirectory(new File(execLocalPath));
+ logger.info("exec local path: {} cleared.", execLocalPath);
+ } catch (IOException e) {
+ logger.error("delete exec dir failed : {}", e.getMessage(), e);
+ }
}
+
}
/**
* get global paras map
- * @return
*/
private Map<String, String> getGlobalParamsMap() {
- Map<String,String> globalParamsMap = new HashMap<>(16);
+ Map<String, String> globalParamsMap = new HashMap<>(16);
// global params string
String globalParamsStr = taskExecutionContext.getGlobalParams();
@@ -168,17 +201,17 @@ public class TaskExecuteThread implements Runnable {
/**
* set task timeout
+ *
* @param taskExecutionContext TaskExecutionContext
- * @param taskNode
*/
private void setTaskTimeout(TaskExecutionContext taskExecutionContext, TaskNode taskNode) {
// the default timeout is the maximum value of the integer
taskExecutionContext.setTaskTimeout(Integer.MAX_VALUE);
TaskTimeoutParameter taskTimeoutParameter = taskNode.getTaskTimeoutParameter();
- if (taskTimeoutParameter.getEnable()){
+ if (taskTimeoutParameter.getEnable()) {
// get timeout strategy
taskExecutionContext.setTaskTimeoutStrategy(taskTimeoutParameter.getStrategy().getCode());
- switch (taskTimeoutParameter.getStrategy()){
+ switch (taskTimeoutParameter.getStrategy()) {
case WARN:
break;
case FAILED:
@@ -199,38 +232,32 @@ public class TaskExecuteThread implements Runnable {
}
}
-
/**
- * kill task
+ * kill task
*/
- public void kill(){
- if (task != null){
+ public void kill() {
+ if (task != null) {
try {
task.cancelApplication(true);
- }catch (Exception e){
- logger.error(e.getMessage(),e);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
}
}
}
-
/**
* download resource file
- *
- * @param execLocalPath
- * @param projectRes
- * @param logger
*/
private void downloadResource(String execLocalPath,
- Map<String,String> projectRes,
+ Map<String, String> projectRes,
Logger logger) throws Exception {
- if (MapUtils.isEmpty(projectRes)){
+ if (MapUtils.isEmpty(projectRes)) {
return;
}
Set<Map.Entry<String, String>> resEntries = projectRes.entrySet();
- for (Map.Entry<String,String> resource : resEntries) {
+ for (Map.Entry<String, String> resource : resEntries) {
String fullName = resource.getKey();
String tenantCode = resource.getValue();
File resFile = new File(execLocalPath, fullName);
@@ -241,8 +268,8 @@ public class TaskExecuteThread implements Runnable {
logger.info("get resource file from hdfs :{}", resHdfsPath);
HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + fullName, false, true);
- }catch (Exception e){
- logger.error(e.getMessage(),e);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage());
}
} else {
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
index a451ce3..6f8f3f9 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
@@ -47,6 +47,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE;
+import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_KILL;
import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS;
/**
@@ -160,12 +161,18 @@ public abstract class AbstractCommandExecutor {
* @return CommandExecuteResult
* @throws Exception if error throws Exception
*/
- public CommandExecuteResult run(String execCommand) throws Exception{
+ public CommandExecuteResult run(String execCommand) throws Exception {
CommandExecuteResult result = new CommandExecuteResult();
-
+ int taskInstanceId = taskExecutionContext.getTaskInstanceId();
+ // If the task has been killed, then the task in the cache is null
+ if (null == taskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId)) {
+ result.setExitStatusCode(EXIT_CODE_KILL);
+ return result;
+ }
if (StringUtils.isEmpty(execCommand)) {
+ taskExecutionContextCacheManager.removeByTaskInstanceId(taskInstanceId);
return result;
}
@@ -187,7 +194,12 @@ public abstract class AbstractCommandExecutor {
// cache processId
taskExecutionContext.setProcessId(processId);
- taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
+ boolean updateTaskExecutionContextStatus = taskExecutionContextCacheManager.updateTaskExecutionContext(taskExecutionContext);
+ if (Boolean.FALSE.equals(updateTaskExecutionContextStatus)) {
+ ProcessUtils.kill(taskExecutionContext);
+ result.setExitStatusCode(EXIT_CODE_KILL);
+ return result;
+ }
// print process id
logger.info("process start, process id is: {}", processId);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
index 075be60..a331765 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
@@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.server.worker.task.shell;
+import static java.util.Calendar.DAY_OF_MONTH;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -40,6 +41,8 @@ import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Date;
+import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -131,25 +134,26 @@ public class ShellTask extends AbstractTask {
* combining local and global parameters
*/
Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
- taskExecutionContext.getDefinedParams(),
- shellParameters.getLocalParametersMap(),
- CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
- taskExecutionContext.getScheduleTime());
- if (paramsMap != null){
- script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
- }
- // new
+ taskExecutionContext.getDefinedParams(),
+ shellParameters.getLocalParametersMap(),
+ CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
+ taskExecutionContext.getScheduleTime());
// replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job
- if (paramsMap != null) {
- if (taskExecutionContext.getScheduleTime() != null) {
- String dateTime = DateUtils.format(taskExecutionContext.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME);
- Property p = new Property();
- p.setValue(dateTime);
- p.setProp(Constants.PARAMETER_SHECDULE_TIME);
- paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
+ if (taskExecutionContext.getScheduleTime() != null) {
+ if (paramsMap == null) {
+ paramsMap = new HashMap<>();
+ }
+ Date date = taskExecutionContext.getScheduleTime();
+ if (CommandType.COMPLEMENT_DATA.getCode() == taskExecutionContext.getCmdTypeIfComplement()) {
+ date = DateUtils.add(taskExecutionContext.getScheduleTime(), DAY_OF_MONTH, 1);
}
- script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap));
+ String dateTime = DateUtils.format(date, Constants.PARAMETER_FORMAT_TIME);
+ Property p = new Property();
+ p.setValue(dateTime);
+ p.setProp(Constants.PARAMETER_DATETIME);
+ paramsMap.put(Constants.PARAMETER_DATETIME, p);
}
+ script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
shellParameters.setRawScript(script);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java
new file mode 100644
index 0000000..772807b
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.task.sqoop;
+
+public final class SqoopConstants {
+
+ private SqoopConstants() {
+ }
+
+ //sqoop general param
+ public static final String SQOOP = "sqoop";
+ public static final String SQOOP_MR_JOB_NAME = "mapred.job.name";
+ public static final String SQOOP_PARALLELISM = "-m";
+ public static final String FIELDS_TERMINATED_BY = "--fields-terminated-by";
+ public static final String LINES_TERMINATED_BY = "--lines-terminated-by";
+ public static final String FIELD_NULL_PLACEHOLDER = "--null-non-string 'NULL' --null-string 'NULL'";
+
+ //sqoop db
+ public static final String DB_CONNECT = "--connect";
+ public static final String DB_USERNAME = "--username";
+ public static final String DB_PWD = "--password";
+ public static final String TABLE = "--table";
+ public static final String COLUMNS = "--columns";
+ public static final String QUERY_WHERE = "where";
+ public static final String QUERY = "--query";
+ public static final String QUERY_CONDITION = "AND \\$CONDITIONS";
+ public static final String QUERY_WITHOUT_CONDITION = "WHERE \\$CONDITIONS";
+ public static final String MAP_COLUMN_HIVE = "--map-column-hive";
+ public static final String MAP_COLUMN_JAVA = "--map-column-java";
+
+
+ //sqoop hive source
+ public static final String HCATALOG_DATABASE = "--hcatalog-database";
+ public static final String HCATALOG_TABLE = "--hcatalog-table";
+ public static final String HCATALOG_PARTITION_KEYS = "--hcatalog-partition-keys";
+ public static final String HCATALOG_PARTITION_VALUES = "--hcatalog-partition-values";
+
+ //sqoop hdfs
+ public static final String HDFS_EXPORT_DIR = "--export-dir";
+ public static final String TARGET_DIR = "--target-dir";
+ public static final String COMPRESSION_CODEC = "--compression-codec";
+
+ //sqoop hive
+ public static final String HIVE_IMPORT = "--hive-import";
+ public static final String HIVE_DATABASE = "--hive-database";
+ public static final String HIVE_TABLE = "--hive-table";
+ public static final String CREATE_HIVE_TABLE = "--create-hive-table";
+ public static final String HIVE_DROP_IMPORT_DELIMS = "--hive-drop-import-delims";
+ public static final String HIVE_OVERWRITE = "--hive-overwrite";
+ public static final String DELETE_TARGET_DIR = "--delete-target-dir";
+ public static final String HIVE_DELIMS_REPLACEMENT = "--hive-delims-replacement";
+ public static final String HIVE_PARTITION_KEY = "--hive-partition-key";
+ public static final String HIVE_PARTITION_VALUE = "--hive-partition-value";
+
+ //sqoop update model
+ public static final String UPDATE_KEY = "--update-key";
+ public static final String UPDATE_MODE = "--update-mode";
+
+
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java
index 9f54d08..00d94f0 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java
@@ -14,63 +14,73 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop;
-import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator;
-import org.slf4j.Logger;
+
import java.util.Map;
+import org.slf4j.Logger;
+
/**
* sqoop task extends the shell task
*/
public class SqoopTask extends AbstractYarnTask {
+ /**
+ * sqoop task params
+ */
private SqoopParameters sqoopParameters;
/**
* taskExecutionContext
*/
- private TaskExecutionContext taskExecutionContext;
+ private final TaskExecutionContext sqoopTaskExecutionContext;
- public SqoopTask(TaskExecutionContext taskExecutionContext, Logger logger){
- super(taskExecutionContext,logger);
- this.taskExecutionContext = taskExecutionContext;
+ public SqoopTask(TaskExecutionContext taskExecutionContext, Logger logger) {
+ super(taskExecutionContext, logger);
+ this.sqoopTaskExecutionContext = taskExecutionContext;
}
@Override
- public void init() throws Exception {
- logger.info("sqoop task params {}", taskExecutionContext.getTaskParams());
+ public void init() {
+ logger.info("sqoop task params {}", sqoopTaskExecutionContext.getTaskParams());
sqoopParameters =
- JSON.parseObject(taskExecutionContext.getTaskParams(),SqoopParameters.class);
- if (!sqoopParameters.checkParameters()) {
- throw new RuntimeException("sqoop task params is not valid");
+ JSONUtils.parseObject(sqoopTaskExecutionContext.getTaskParams(), SqoopParameters.class);
+ //check sqoop task params
+ if (null == sqoopParameters) {
+ throw new IllegalArgumentException("Sqoop Task params is null");
}
+ if (!sqoopParameters.checkParameters()) {
+ throw new IllegalArgumentException("Sqoop Task params check fail");
+ }
}
@Override
- protected String buildCommand() throws Exception {
+ protected String buildCommand() {
//get sqoop scripts
SqoopJobGenerator generator = new SqoopJobGenerator();
- String script = generator.generateSqoopJob(sqoopParameters,taskExecutionContext);
+ String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext);
- Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
- taskExecutionContext.getDefinedParams(),
- sqoopParameters.getLocalParametersMap(),
- CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
- taskExecutionContext.getScheduleTime());
+ Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(sqoopTaskExecutionContext.getDefinedParams()),
+ sqoopTaskExecutionContext.getDefinedParams(),
+ sqoopParameters.getLocalParametersMap(),
+ CommandType.of(sqoopTaskExecutionContext.getCmdTypeIfComplement()),
+ sqoopTaskExecutionContext.getScheduleTime());
- if(paramsMap != null){
- String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
+ if (paramsMap != null) {
+ String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
logger.info("sqoop script: {}", resultScripts);
return resultScripts;
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java
index 4944bac..6986798 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java
@@ -14,9 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
+
+import java.util.List;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,21 +33,54 @@ import org.slf4j.LoggerFactory;
*/
public class CommonGenerator {
- private Logger logger = LoggerFactory.getLogger(getClass());
+ private static final Logger logger = LoggerFactory.getLogger(CommonGenerator.class);
public String generate(SqoopParameters sqoopParameters) {
- StringBuilder result = new StringBuilder();
- try{
- result.append("sqoop ")
- .append(sqoopParameters.getModelType());
- if(sqoopParameters.getConcurrency() >0){
- result.append(" -m ")
- .append(sqoopParameters.getConcurrency());
+
+ StringBuilder commonSb = new StringBuilder();
+
+ try {
+ //sqoop task model
+ commonSb.append(SqoopConstants.SQOOP)
+ .append(Constants.SPACE)
+ .append(sqoopParameters.getModelType());
+
+ //sqoop map-reduce job name
+ commonSb.append(Constants.SPACE).append(Constants.D).append(Constants.SPACE)
+ .append(String.format("%s%s%s", SqoopConstants.SQOOP_MR_JOB_NAME,
+ Constants.EQUAL_SIGN, sqoopParameters.getJobName()));
+
+ //hadoop custom param
+ List<Property> hadoopCustomParams = sqoopParameters.getHadoopCustomParams();
+ if (CollectionUtils.isNotEmpty(hadoopCustomParams)) {
+ StringBuilder hadoopCustomParamStr = new StringBuilder();
+ for (Property hadoopCustomParam : hadoopCustomParams) {
+ hadoopCustomParamStr.append(Constants.D)
+ .append(Constants.SPACE).append(hadoopCustomParam.getProp())
+ .append(Constants.EQUAL_SIGN).append(hadoopCustomParam.getValue())
+ .append(Constants.SPACE);
+ }
+ commonSb.append(Constants.SPACE).append(hadoopCustomParamStr.substring(0, hadoopCustomParamStr.length() - 1));
+ }
+
+ //sqoop custom params
+ List<Property> sqoopAdvancedParams = sqoopParameters.getSqoopAdvancedParams();
+ if (CollectionUtils.isNotEmpty(sqoopAdvancedParams)) {
+ for (Property sqoopAdvancedParam : sqoopAdvancedParams) {
+ commonSb.append(Constants.SPACE).append(sqoopAdvancedParam.getProp())
+ .append(Constants.SPACE).append(sqoopAdvancedParam.getValue());
+ }
+ }
+
+ //sqoop parallelism
+ if (sqoopParameters.getConcurrency() > 0) {
+ commonSb.append(Constants.SPACE).append(SqoopConstants.SQOOP_PARALLELISM)
+ .append(Constants.SPACE).append(sqoopParameters.getConcurrency());
}
- }catch (Exception e){
- logger.error(e.getMessage());
+ } catch (Exception e) {
+ logger.error(String.format("Sqoop task general param build failed: [%s]", e.getMessage()));
}
- return result.toString();
+ return commonSb.toString();
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java
index 841654b..e6a9576 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
@@ -26,9 +27,10 @@ public interface ISourceGenerator {
/**
* generate the source script
- * @param sqoopParameters sqoopParameters
+ *
+ * @param sqoopParameters sqoopParameters
* @param taskExecutionContext taskExecutionContext
* @return source script
*/
- String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext);
+ String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext);
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java
index 7bdaf49..eb355a7 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
@@ -26,9 +27,10 @@ public interface ITargetGenerator {
/**
* generate the target script
- * @param sqoopParameters sqoopParameters
+ *
+ * @param sqoopParameters sqoopParameters
* @param taskExecutionContext taskExecutionContext
* @return target script
*/
- String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext);
+ String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext);
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java
index 4e9cb84..9feaffa 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java
@@ -14,8 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
+import org.apache.dolphinscheduler.common.enums.SqoopJobType;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources.HdfsSourceGenerator;
@@ -45,40 +47,51 @@ public class SqoopJobGenerator {
/**
* common script generator
*/
- private CommonGenerator commonGenerator;
+ private final CommonGenerator commonGenerator;
- public SqoopJobGenerator(){
+ public SqoopJobGenerator() {
commonGenerator = new CommonGenerator();
}
- private void createSqoopJobGenerator(String sourceType,String targetType){
+ private void createSqoopJobGenerator(String sourceType, String targetType) {
sourceGenerator = createSourceGenerator(sourceType);
targetGenerator = createTargetGenerator(targetType);
}
/**
* get the final sqoop scripts
- * @param sqoopParameters
- * @return
+ *
+ * @param sqoopParameters sqoop params
+ * @return sqoop scripts
*/
- public String generateSqoopJob(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext){
- createSqoopJobGenerator(sqoopParameters.getSourceType(),sqoopParameters.getTargetType());
- if(sourceGenerator == null || targetGenerator == null){
- return null;
+ public String generateSqoopJob(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+ String sqoopScripts = "";
+
+ if (SqoopJobType.TEMPLATE.getDescp().equals(sqoopParameters.getJobType())) {
+ createSqoopJobGenerator(sqoopParameters.getSourceType(), sqoopParameters.getTargetType());
+ if (sourceGenerator == null || targetGenerator == null) {
+ throw new RuntimeException("sqoop task source type or target type is null");
+ }
+
+ sqoopScripts = String.format("%s%s%s", commonGenerator.generate(sqoopParameters),
+ sourceGenerator.generate(sqoopParameters, taskExecutionContext),
+ targetGenerator.generate(sqoopParameters, taskExecutionContext));
+ } else if (SqoopJobType.CUSTOM.getDescp().equals(sqoopParameters.getJobType())) {
+ sqoopScripts = sqoopParameters.getCustomShell().replaceAll("\\r\\n", "\n");
}
- return commonGenerator.generate(sqoopParameters)
- + sourceGenerator.generate(sqoopParameters,taskExecutionContext)
- + targetGenerator.generate(sqoopParameters,taskExecutionContext);
+ return sqoopScripts;
}
/**
* get the source generator
- * @param sourceType
- * @return
+ *
+ * @param sourceType sqoop source type
+ * @return sqoop source generator
*/
- private ISourceGenerator createSourceGenerator(String sourceType){
- switch (sourceType){
+ private ISourceGenerator createSourceGenerator(String sourceType) {
+ switch (sourceType) {
case MYSQL:
return new MysqlSourceGenerator();
case HIVE:
@@ -92,11 +105,12 @@ public class SqoopJobGenerator {
/**
* get the target generator
- * @param targetType
- * @return
+ *
+ * @param targetType sqoop target type
+ * @return sqoop target generator
*/
- private ITargetGenerator createTargetGenerator(String targetType){
- switch (targetType){
+ private ITargetGenerator createTargetGenerator(String targetType) {
+ switch (targetType) {
case MYSQL:
return new MysqlTargetGenerator();
case HIVE:
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java
index 41e5668..549d5db 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java
@@ -14,14 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
+import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,28 +34,30 @@ import org.slf4j.LoggerFactory;
*/
public class HdfsSourceGenerator implements ISourceGenerator {
- private Logger logger = LoggerFactory.getLogger(getClass());
+ private static final Logger logger = LoggerFactory.getLogger(HdfsSourceGenerator.class);
@Override
- public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
- StringBuilder result = new StringBuilder();
- try{
+ public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+ StringBuilder hdfsSourceSb = new StringBuilder();
+
+ try {
SourceHdfsParameter sourceHdfsParameter
- = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceHdfsParameter.class);
-
- if(sourceHdfsParameter != null){
- if(StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())){
- result.append(" --export-dir ")
- .append(sourceHdfsParameter.getExportDir());
- }else{
- throw new Exception("--export-dir is null");
+ = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHdfsParameter.class);
+
+ if (null != sourceHdfsParameter) {
+ if (StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())) {
+ hdfsSourceSb.append(Constants.SPACE).append(SqoopConstants.HDFS_EXPORT_DIR)
+ .append(Constants.SPACE).append(sourceHdfsParameter.getExportDir());
+ } else {
+ throw new IllegalArgumentException("Sqoop hdfs export dir is null");
}
}
- }catch (Exception e){
- logger.error("get hdfs source failed",e);
+ } catch (Exception e) {
+ logger.error(String.format("Sqoop hdfs source parmas build failed: [%s]", e.getMessage()));
}
- return result.toString();
+ return hdfsSourceSb.toString();
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java
index ea12616..3229dca 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java
@@ -14,14 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,33 +34,40 @@ import org.slf4j.LoggerFactory;
*/
public class HiveSourceGenerator implements ISourceGenerator {
- private Logger logger = LoggerFactory.getLogger(getClass());
+ private static final Logger logger = LoggerFactory.getLogger(HiveSourceGenerator.class);
@Override
- public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
- StringBuilder sb = new StringBuilder();
- try{
+ public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+ StringBuilder hiveSourceSb = new StringBuilder();
+
+ try {
SourceHiveParameter sourceHiveParameter
- = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceHiveParameter.class);
- if(sourceHiveParameter != null){
- if(StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())){
- sb.append(" --hcatalog-database ").append(sourceHiveParameter.getHiveDatabase());
+ = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHiveParameter.class);
+
+ if (null != sourceHiveParameter) {
+ if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())) {
+ hiveSourceSb.append(Constants.SPACE).append(SqoopConstants.HCATALOG_DATABASE)
+ .append(Constants.SPACE).append(sourceHiveParameter.getHiveDatabase());
}
- if(StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())){
- sb.append(" --hcatalog-table ").append(sourceHiveParameter.getHiveTable());
+ if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())) {
+ hiveSourceSb.append(Constants.SPACE).append(SqoopConstants.HCATALOG_TABLE)
+ .append(Constants.SPACE).append(sourceHiveParameter.getHiveTable());
}
- if(StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey())&&
- StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())){
- sb.append(" --hcatalog-partition-keys ").append(sourceHiveParameter.getHivePartitionKey())
- .append(" --hcatalog-partition-values ").append(sourceHiveParameter.getHivePartitionValue());
+ if (StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey())
+ && StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())) {
+ hiveSourceSb.append(Constants.SPACE).append(SqoopConstants.HCATALOG_PARTITION_KEYS)
+ .append(Constants.SPACE).append(sourceHiveParameter.getHivePartitionKey())
+ .append(Constants.SPACE).append(SqoopConstants.HCATALOG_PARTITION_VALUES)
+ .append(Constants.SPACE).append(sourceHiveParameter.getHivePartitionValue());
}
}
- }catch (Exception e){
- logger.error(e.getMessage());
+ } catch (Exception e) {
+ logger.error(String.format("Sqoop hive source params build failed: [%s]", e.getMessage()));
}
- return sb.toString();
+ return hiveSourceSb.toString();
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java
index 8cd5357..f0b24a9 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java
@@ -14,106 +14,124 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
-import org.apache.dolphinscheduler.common.enums.QueryType;
+import org.apache.dolphinscheduler.common.enums.SqoopQueryType;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.server.entity.SqoopTaskExecutionContext;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* mysql source generator
*/
public class MysqlSourceGenerator implements ISourceGenerator {
- private Logger logger = LoggerFactory.getLogger(getClass());
+ private static final Logger logger = LoggerFactory.getLogger(MysqlSourceGenerator.class);
@Override
- public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
- StringBuilder result = new StringBuilder();
- try {
- SourceMysqlParameter sourceMysqlParameter
- = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceMysqlParameter.class);
+ public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+ StringBuilder mysqlSourceSb = new StringBuilder();
+ try {
+ SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class);
SqoopTaskExecutionContext sqoopTaskExecutionContext = taskExecutionContext.getSqoopTaskExecutionContext();
- if(sourceMysqlParameter != null){
+ if (null != sourceMysqlParameter) {
BaseDataSource baseDataSource = DataSourceFactory.getDatasource(DbType.of(sqoopTaskExecutionContext.getSourcetype()),
- sqoopTaskExecutionContext.getSourceConnectionParams());
- if(baseDataSource != null){
- result.append(" --connect ")
- .append(baseDataSource.getJdbcUrl())
- .append(" --username ")
- .append(baseDataSource.getUser())
- .append(" --password ")
- .append(baseDataSource.getPassword());
-
- if(sourceMysqlParameter.getSrcQueryType() == QueryType.FORM.ordinal()){
- if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())){
- result.append(" --table ").append(sourceMysqlParameter.getSrcTable());
+ sqoopTaskExecutionContext.getSourceConnectionParams());
+
+ if (null != baseDataSource) {
+
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.DB_CONNECT)
+ .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(baseDataSource.getJdbcUrl()).append(Constants.DOUBLE_QUOTES)
+ .append(Constants.SPACE).append(SqoopConstants.DB_USERNAME)
+ .append(Constants.SPACE).append(baseDataSource.getUser())
+ .append(Constants.SPACE).append(SqoopConstants.DB_PWD)
+ .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(baseDataSource.getPassword()).append(Constants.DOUBLE_QUOTES);
+
+ //sqoop table & sql query
+ if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.FORM.getCode()) {
+ if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())) {
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.TABLE)
+ .append(Constants.SPACE).append(sourceMysqlParameter.getSrcTable());
}
- if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())){
- result.append(" --columns ").append(sourceMysqlParameter.getSrcColumns());
+ if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())) {
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.COLUMNS)
+ .append(Constants.SPACE).append(sourceMysqlParameter.getSrcColumns());
}
+ } else if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.SQL.getCode()
+ && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())) {
- }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal()
- && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){
String srcQuery = sourceMysqlParameter.getSrcQuerySql();
- if(srcQuery.toLowerCase().contains("where")){
- srcQuery += " AND "+"$CONDITIONS";
- }else{
- srcQuery += " WHERE $CONDITIONS";
- }
- result.append(" --query \'").append(srcQuery).append("\'");
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.QUERY)
+ .append(Constants.SPACE).append(Constants.DOUBLE_QUOTES).append(srcQuery);
+ if (srcQuery.toLowerCase().contains(SqoopConstants.QUERY_WHERE)) {
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.QUERY_CONDITION).append(Constants.DOUBLE_QUOTES);
+ } else {
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.QUERY_WITHOUT_CONDITION).append(Constants.DOUBLE_QUOTES);
+ }
}
- List<Property> mapColumnHive = sourceMysqlParameter.getMapColumnHive();
+ //sqoop hive map column
+ List<Property> mapColumnHive = sourceMysqlParameter.getMapColumnHive();
- if(mapColumnHive != null && !mapColumnHive.isEmpty()){
+ if (null != mapColumnHive && !mapColumnHive.isEmpty()) {
StringBuilder columnMap = new StringBuilder();
- for(Property item:mapColumnHive){
- columnMap.append(item.getProp()).append("=").append(item.getValue()).append(",");
+ for (Property item : mapColumnHive) {
+ if (!item.getProp().isEmpty()) {
+ columnMap.append(item.getProp()).append(Constants.EQUAL_SIGN)
+ .append(item.getValue()).append(Constants.COMMA);
+ }
}
- if(StringUtils.isNotEmpty(columnMap.toString())){
- result.append(" --map-column-hive ")
- .append(columnMap.substring(0,columnMap.length() - 1));
+ if (StringUtils.isNotEmpty(columnMap.toString())) {
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.MAP_COLUMN_HIVE)
+ .append(Constants.SPACE).append(columnMap.substring(0, columnMap.length() - 1));
}
}
- List<Property> mapColumnJava = sourceMysqlParameter.getMapColumnJava();
+ //sqoop map column java
+ List<Property> mapColumnJava = sourceMysqlParameter.getMapColumnJava();
- if(mapColumnJava != null && !mapColumnJava.isEmpty()){
- StringBuilder columnMap = new StringBuilder();
- for(Property item:mapColumnJava){
- columnMap.append(item.getProp()).append("=").append(item.getValue()).append(",");
+ if (null != mapColumnJava && !mapColumnJava.isEmpty()) {
+ StringBuilder columnJavaMap = new StringBuilder();
+ for (Property item : mapColumnJava) {
+ if (!item.getProp().isEmpty()) {
+ columnJavaMap.append(item.getProp()).append(Constants.EQUAL_SIGN)
+ .append(item.getValue()).append(Constants.COMMA);
+ }
}
- if(StringUtils.isNotEmpty(columnMap.toString())){
- result.append(" --map-column-java ")
- .append(columnMap.substring(0,columnMap.length() - 1));
+ if (StringUtils.isNotEmpty(columnJavaMap.toString())) {
+ mysqlSourceSb.append(Constants.SPACE).append(SqoopConstants.MAP_COLUMN_JAVA)
+ .append(Constants.SPACE).append(columnJavaMap.substring(0, columnJavaMap.length() - 1));
}
}
}
}
- }catch (Exception e){
- logger.error(e.getMessage());
+ } catch (Exception e) {
+ logger.error(String.format("Sqoop task mysql source params build failed: [%s]", e.getMessage()));
}
- return result.toString();
+ return mysqlSourceSb.toString();
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java
index 64ea75e..3ea3254 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java
@@ -14,14 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,47 +34,53 @@ import org.slf4j.LoggerFactory;
*/
public class HdfsTargetGenerator implements ITargetGenerator {
- private Logger logger = LoggerFactory.getLogger(getClass());
+ private static final Logger logger = LoggerFactory.getLogger(HdfsTargetGenerator.class);
@Override
- public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
- StringBuilder result = new StringBuilder();
- try{
+ public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
+
+ StringBuilder hdfsTargetSb = new StringBuilder();
+
+ try {
TargetHdfsParameter targetHdfsParameter =
- JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetHdfsParameter.class);
+ JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHdfsParameter.class);
- if(targetHdfsParameter != null){
+ if (null != targetHdfsParameter) {
- if(StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())){
- result.append(" --target-dir ").append(targetHdfsParameter.getTargetPath());
+ if (StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())) {
+ hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.TARGET_DIR)
+ .append(Constants.SPACE).append(targetHdfsParameter.getTargetPath());
}
- if(StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())){
- result.append(" --compression-codec ").append(targetHdfsParameter.getCompressionCodec());
+ if (StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())) {
+ hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.COMPRESSION_CODEC)
+ .append(Constants.SPACE).append(targetHdfsParameter.getCompressionCodec());
}
- if(StringUtils.isNotEmpty(targetHdfsParameter.getFileType())){
- result.append(" ").append(targetHdfsParameter.getFileType());
+ if (StringUtils.isNotEmpty(targetHdfsParameter.getFileType())) {
+ hdfsTargetSb.append(Constants.SPACE).append(targetHdfsParameter.getFileType());
}
- if(targetHdfsParameter.isDeleteTargetDir()){
- result.append(" --delete-target-dir");
+ if (targetHdfsParameter.isDeleteTargetDir()) {
+ hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.DELETE_TARGET_DIR);
}
- if(StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())){
- result.append(" --fields-terminated-by '").append(targetHdfsParameter.getFieldsTerminated()).append("'");
+ if (StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())) {
+ hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.FIELDS_TERMINATED_BY)
+ .append(Constants.SPACE).append(Constants.SINGLE_QUOTES).append(targetHdfsParameter.getFieldsTerminated()).append(Constants.SINGLE_QUOTES);
}
- if(StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())){
- result.append(" --lines-terminated-by '").append(targetHdfsParameter.getLinesTerminated()).append("'");
+ if (StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())) {
+ hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.LINES_TERMINATED_BY)
+ .append(Constants.SPACE).append(Constants.SINGLE_QUOTES).append(targetHdfsParameter.getLinesTerminated()).append(Constants.SINGLE_QUOTES);
}
- result.append(" --null-non-string 'NULL' --null-string 'NULL'");
+ hdfsTargetSb.append(Constants.SPACE).append(SqoopConstants.FIELD_NULL_PLACEHOLDER);
}
- }catch(Exception e){
- logger.error(e.getMessage());
+ } catch (Exception e) {
+ logger.error(String.format("Sqoop hdfs target params build failed: [%s]", e.getMessage()));
}
- return result.toString();
+ return hdfsTargetSb.toString();
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java
index dc5440b..ae91517 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java
@@ -14,14 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
-import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopConstants;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,57 +34,63 @@ import org.slf4j.LoggerFactory;
*/
public class HiveTargetGenerator implements ITargetGenerator {
- private Logger logger = LoggerFactory.getLogger(getClass());
+ private static final Logger logger = LoggerFactory.getLogger(HiveTargetGenerator.class);
@Override
- public String generate(SqoopParameters sqoopParameters,TaskExecutionContext taskExecutionContext) {
+ public String generate(SqoopParameters sqoopParameters, TaskExecutionContext taskExecutionContext) {
- StringBuilder result = new StringBuilder();
+ StringBuilder hiveTargetSb = new StringBuilder();
- try{
+ try {
TargetHiveParameter targetHiveParameter =
- JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetHiveParameter.class);
- if(targetHiveParameter != null){
+ JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHiveParameter.class);
+ if (null != targetHiveParameter) {
+ hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_IMPORT);
- result.append(" --hive-import ");
+ if (StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase())
+ && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) {
+ hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_DATABASE)
+ .append(Constants.SPACE).append(targetHiveParameter.getHiveDatabase())
+ .append(Constants.SPACE).append(SqoopConstants.HIVE_TABLE)
+ .append(Constants.SPACE).append(targetHiveParameter.getHiveTable());
+ }
- if(StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase())&&
- StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())){
- result.append(" --hive-table ")
- .append(targetHiveParameter.getHiveDatabase())
- .append(".")
- .append(targetHiveParameter.getHiveTable());
+ if (targetHiveParameter.isCreateHiveTable()) {
+ hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.CREATE_HIVE_TABLE);
}
... 3311 lines suppressed ...