You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by le...@apache.org on 2020/06/30 06:20:59 UTC
[incubator-dolphinscheduler] branch 1.3.1-release updated (58897aa
-> 3485f45)
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a change to branch 1.3.1-release
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git.
from 58897aa merge 1.3.0-release branch code
new 7445fa9 fix singe resource manager will NPE (#3013)
new 0f8cb38 [bugfix] remove Conflicting configuration sudo and user (#3038)
new 2031528 [bugfix] 1. fix alert.json includes repeated tag 'DOLPHIN_ALERT' 2. metainfo.xml The version number of the RPM package is no longer specified in the metainfo.xml (#3032)
new 3485f45 [feature] readme.md adds instructions for changing environment variables (#3031)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ambari_plugin/README.md | 25 ++++++---
.../common-services/DOLPHIN/1.3.0/alerts.json | 60 ++++++++++------------
.../common-services/DOLPHIN/1.3.0/metainfo.xml | 4 +-
.../1.3.0/package/scripts/dolphin_alert_service.py | 3 +-
.../1.3.0/package/scripts/dolphin_api_service.py | 3 +-
.../package/scripts/dolphin_logger_service.py | 4 +-
.../package/scripts/dolphin_master_service.py | 3 +-
.../package/scripts/dolphin_worker_service.py | 3 +-
.../dolphinscheduler/common/utils/HadoopUtils.java | 20 ++++----
.../src/main/resources/common.properties | 2 +-
.../common/utils/HadoopUtilsTest.java | 4 +-
.../src/main/resources/config/install_config.conf | 2 +-
12 files changed, 70 insertions(+), 63 deletions(-)
[incubator-dolphinscheduler] 03/04: [bugfix] 1. fix alert.json
includes repeated tag 'DOLPHIN_ALERT' 2. metainfo.xml The version number of
the RPM package is no longer specified in the metainfo.xml (#3032)
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 1.3.1-release
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
commit 2031528addbdb5c9503932450f223bf6399abf18
Author: zhangchunyang <zh...@gmail.com>
AuthorDate: Tue Jun 30 10:44:52 2020 +0800
[bugfix] 1. fix alert.json includes repeated tag 'DOLPHIN_ALERT' 2. metainfo.xml The version number of the RPM package is no longer specified in the metainfo.xml (#3032)
Co-authored-by: zhangchunyang1024 <zh...@gamil.com>
---
.../common-services/DOLPHIN/1.3.0/alerts.json | 60 ++++++++++------------
.../common-services/DOLPHIN/1.3.0/metainfo.xml | 4 +-
2 files changed, 29 insertions(+), 35 deletions(-)
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json b/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json
index 2a9800e..4949f7d 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/alerts.json
@@ -28,6 +28,33 @@
}
}
],
+ "DOLPHIN_LOGGER": [
+ {
+ "name": "dolphin_logger_port_check",
+ "label": "dolphin_logger_port_check",
+ "description": "dolphin_logger_port_check.",
+ "interval": 10,
+ "scope": "ANY",
+ "source": {
+ "type": "PORT",
+ "uri": "{{dolphin-common/loggerserver.rpc.port}}",
+ "default_port": 50051,
+ "reporting": {
+ "ok": {
+ "text": "TCP OK - {0:.3f}s response on port {1}"
+ },
+ "warning": {
+ "text": "TCP OK - {0:.3f}s response on port {1}",
+ "value": 1.5
+ },
+ "critical": {
+ "text": "Connection failed: {0} to {1}:{2}",
+ "value": 5.0
+ }
+ }
+ }
+ }
+ ],
"DOLPHIN_MASTER": [
{
"name": "DOLPHIN_MASTER_CHECK",
@@ -126,39 +153,6 @@
]
}
}
- ],
- "DOLPHIN_ALERT": [
- {
- "name": "DOLPHIN_DOLPHIN_LOGGER_CHECK",
- "label": "check dolphin scheduler alert status",
- "description": "",
- "interval":10,
- "scope": "HOST",
- "enabled": true,
- "source": {
- "type": "SCRIPT",
- "path": "DOLPHIN/1.3.0/package/alerts/alert_dolphin_scheduler_status.py",
- "parameters": [
-
- {
- "name": "connection.timeout",
- "display_name": "Connection Timeout",
- "value": 5.0,
- "type": "NUMERIC",
- "description": "The maximum time before this alert is considered to be CRITICAL",
- "units": "seconds",
- "threshold": "CRITICAL"
- },
- {
- "name": "alertName",
- "display_name": "alertName",
- "value": "DOLPHIN_LOGGER",
- "type": "STRING",
- "description": "alert name"
- }
- ]
- }
- }
]
}
}
\ No newline at end of file
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml
index a559085..183221a 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml
@@ -103,7 +103,7 @@
<osFamily>any</osFamily>
<packages>
<package>
- <name>apache-dolphinscheduler-incubating-1.3.0*</name>
+ <name>apache-dolphinscheduler-incubating*</name>
</package>
</packages>
</osSpecific>
@@ -134,4 +134,4 @@
</quickLinksConfigurations>
</service>
</services>
-</metainfo>
+</metainfo>
\ No newline at end of file
[incubator-dolphinscheduler] 01/04: fix singe resource manager will
NPE (#3013)
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 1.3.1-release
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
commit 7445fa946bfcaf2c2da7c0bd1536ac979bf1d240
Author: dailidong <da...@gmail.com>
AuthorDate: Fri Jun 19 14:45:48 2020 +0800
fix singe resource manager will NPE (#3013)
* fix worker group config no effect
* remove codehaus janino jar
the license about janino maybe not compatiable with Apache v2
* Merge remote-tracking branch 'upstream/dev-1.3.0' into dev-1.3.0
# Conflicts:
# dolphinscheduler-server/src/main/resources/config/install_config.conf
* datasource config
* Update datasource.properties
* fix RunConfig bug
* remove param monitor server state
* fix table T_DS_ALERT
* update h2 database
* fix #2910 master server will show exception for some time when it restart
* fix oom when no master is active
* fix worker oom when master server restart
* fix oom
* fix
* add UT
* fix worker group config no effect
* fix singe resource manager will NPE
* Update HadoopUtilsTest.java
---
.../dolphinscheduler/common/utils/HadoopUtils.java | 20 ++++++++++----------
.../src/main/resources/common.properties | 2 +-
.../common/utils/HadoopUtilsTest.java | 4 ++--
.../src/main/resources/config/install_config.conf | 2 +-
4 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
index 963aff5..b4eebd6 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
@@ -186,7 +186,7 @@ public class HadoopUtils implements Closeable {
* @param applicationId application id
* @return url of application
*/
- public String getApplicationUrl(String applicationId) {
+ public String getApplicationUrl(String applicationId) throws Exception {
/**
* if rmHaIds contains xx, it signs not use resourcemanager
* otherwise:
@@ -194,21 +194,21 @@ public class HadoopUtils implements Closeable {
* if rmHaIds not empty: resourcemanager HA enabled
*/
String appUrl = "";
- //not use resourcemanager
- if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){
- yarnEnabled = false;
- logger.warn("should not step here");
- } else if (!StringUtils.isEmpty(rmHaIds)) {
+ if (StringUtils.isEmpty(rmHaIds)){
+ //single resourcemanager enabled
+ appUrl = appAddress;
+ yarnEnabled = true;
+ } else {
//resourcemanager HA enabled
appUrl = getAppAddress(appAddress, rmHaIds);
yarnEnabled = true;
logger.info("application url : {}", appUrl);
- } else {
- //single resourcemanager enabled
- yarnEnabled = true;
}
+ if(StringUtils.isBlank(appUrl)){
+ throw new Exception("application url is blank");
+ }
return String.format(appUrl, applicationId);
}
@@ -402,7 +402,7 @@ public class HadoopUtils implements Closeable {
* @return the return may be null or there may be other parse exceptions
* @throws JSONException json exception
*/
- public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException {
+ public ExecutionStatus getApplicationStatus(String applicationId) throws Exception {
if (StringUtils.isEmpty(applicationId)) {
return null;
}
diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
index 923f02a..ef49a19 100644
--- a/dolphinscheduler-common/src/main/resources/common.properties
+++ b/dolphinscheduler-common/src/main/resources/common.properties
@@ -54,7 +54,7 @@ fs.s3a.access.key=A3DXS30FO22544RE
# if resource.storage.type=S3，s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
-# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
+# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname.
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
index e239fe7..a002c79 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
@@ -142,7 +142,7 @@ public class HadoopUtilsTest {
@Test
public void isYarnEnabled() {
boolean result = hadoopUtils.isYarnEnabled();
- Assert.assertEquals(false, result);
+ Assert.assertEquals(true, result);
}
@Test
@@ -185,7 +185,7 @@ public class HadoopUtilsTest {
}
@Test
- public void getApplicationUrl(){
+ public void getApplicationUrl() throws Exception {
String application_1516778421218_0042 = hadoopUtils.getApplicationUrl("application_1529051418016_0167");
logger.info(application_1516778421218_0042);
}
diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf
index 05a3e99..eeff030 100644
--- a/dolphinscheduler-server/src/main/resources/config/install_config.conf
+++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf
@@ -87,7 +87,7 @@ s3Endpoint="http://192.168.xx.xx:9010"
s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx"
-# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
+# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarnHaIps="192.168.xx.xx,192.168.xx.xx"
# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.
[incubator-dolphinscheduler] 04/04: [feature] readme.md adds
instructions for changing environment variables (#3031)
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 1.3.1-release
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
commit 3485f45db5e1c4739cd3a65ab5bece9a8b9c4bc7
Author: zhangchunyang <zh...@gmail.com>
AuthorDate: Tue Jun 30 11:02:25 2020 +0800
[feature] readme.md adds instructions for changing environment variables (#3031)
Co-authored-by: zhangchunyang1024 <zh...@gamil.com>
---
ambari_plugin/README.md | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
diff --git a/ambari_plugin/README.md b/ambari_plugin/README.md
index 817b7ab..1778266 100644
--- a/ambari_plugin/README.md
+++ b/ambari_plugin/README.md
@@ -3,7 +3,8 @@
#### Note
1. This document is intended for users with a basic understanding of Ambari
-2. This document is a description of adding the Dolphin Scheduler(version 1.3.0) service to the installed Ambari service
+2. This document is a description of adding the Dolphin Scheduler service to the installed Ambari service
+3. This document is based on version 2.5.2 of Ambari
#### Installation preparation
@@ -16,8 +17,12 @@
3. Install with rpm package
- Manual installation (recommended）：
- - Copy the prepared RPM packages to each node of the cluster
- - Execute with DS installation user: ```rpm -ivh apache-dolphinscheduler-incubating-xxx.noarch.rpm```
+ - Copy the prepared RPM packages to each node of the cluster.
+ - Execute with DS installation user: ```rpm -ivh apache-dolphinscheduler-incubating-xxx.noarch.rpm```
+ - Mysql-connector-java packaged using the default POM file will not be included.
+ - The RPM package was packaged in the project with the installation path of /opt/soft.
+ If you use mysql as the database, you need add it manually.
+
- Automatic installation with ambari
- Each node of the cluster needs to configure the local yum source
- Copy the prepared RPM packages to each node local yum source
@@ -45,6 +50,7 @@
#### Ambari Install Dolphin Scheduler
+- **NOTE: You have to install zookeeper first**
1. Install Dolphin Scheduler on ambari web interface
@@ -62,21 +68,25 @@
-5. Configure the database information (same as in the initialization database in step 1)
+5. System Env Optimization will export some system environment config. Modify according to actual situation
+
+ 
+
+6. Configure the database information (same as in the initialization database in step 1)
-6. Configure additional information if needed
+7. Configure additional information if needed
-7. Perform the next steps as normal
+8. Perform the next steps as normal
-8. The interface after successful installation
+9. The interface after successful installation
@@ -111,7 +121,6 @@
-
#### Remove the component from the node with Ambari
1. Stop the component in the corresponding node
[incubator-dolphinscheduler] 02/04: [bugfix] remove Conflicting
configuration sudo and user (#3038)
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 1.3.1-release
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
commit 0f8cb383a32974435a6448de30b8cc0f8e979301
Author: zhangchunyang <zh...@gmail.com>
AuthorDate: Sun Jun 28 11:37:59 2020 +0800
[bugfix] remove Conflicting configuration sudo and user (#3038)
Co-authored-by: zhangchunyang1024 <zh...@gamil.com>
---
.../DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py | 3 ++-
.../DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py | 3 ++-
.../DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py | 4 ++--
.../DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py | 3 ++-
.../DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py | 3 ++-
5 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py
index 62255a3..e78c38d 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_alert_service.py
@@ -26,7 +26,8 @@ class DolphinAlertService(Script):
import params
env.set_params(params)
self.install_packages(env)
- Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py
index bdc18fb..5a28924 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_api_service.py
@@ -26,7 +26,8 @@ class DolphinApiService(Script):
import params
env.set_params(params)
self.install_packages(env)
- Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py
index f1c19bd..fb47e13 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_logger_service.py
@@ -26,8 +26,8 @@ class DolphinLoggerService(Script):
import params
env.set_params(params)
self.install_packages(env)
- Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
-
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params
params.pika_slave = True
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py
index 6ee7ecf..8d64935 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_master_service.py
@@ -27,7 +27,8 @@ class DolphinMasterService(Script):
import params
env.set_params(params)
self.install_packages(env)
- Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py
index 2d145ee..1f542c0 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/dolphin_worker_service.py
@@ -26,7 +26,8 @@ class DolphinWorkerService(Script):
import params
env.set_params(params)
self.install_packages(env)
- Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
+ Execute(('chmod', '-R', '777', params.dolphin_home))
+ Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params