You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ki...@apache.org on 2021/07/20 06:29:08 UTC
[dolphinscheduler] branch 1.3.7-prepare updated: Revert
"[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)"
(#5861)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch 1.3.7-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.7-prepare by this push:
new bb4a59f Revert "[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)" (#5861)
bb4a59f is described below
commit bb4a59f10fb7646c9aa8b58d8dc182c5253cc778
Author: Kirs <ac...@163.com>
AuthorDate: Tue Jul 20 14:29:02 2021 +0800
Revert "[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)" (#5861)
This reverts commit 8e2c312d4e8e4d9644fda4be6113ca1dd98884d4.
---
.../apache/dolphinscheduler/common/Constants.java | 9 ------
.../dolphinscheduler/common/utils/HadoopUtils.java | 37 ++++++++++++----------
.../src/main/resources/common.properties | 4 +--
3 files changed, 21 insertions(+), 29 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index e8447fa..fe83fa1 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -104,11 +104,6 @@ public final class Constants {
public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address";
/**
- * yarn.job.history.status.address
- */
- public static final String YARN_JOB_HISTORY_STATUS_ADDRESS = "yarn.job.history.status.address";
-
- /**
* hdfs configuration
* hdfs.root.user
*/
@@ -797,10 +792,6 @@ public final class Constants {
*/
public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state";
- /**
- * com.amazonaws.services.s3.enableV4
- */
- public static final String AWS_S3_V4 = "com.amazonaws.services.s3.enableV4";
/**
* loginUserFromKeytab user
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
index 667838d..ffead0b 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
@@ -70,7 +70,6 @@ public class HadoopUtils implements Closeable {
public static final String resourceUploadPath = PropertyUtils.getString(RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
public static final String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS);
public static final String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS);
- public static final String jobHistoryAddress = PropertyUtils.getString(Constants.YARN_JOB_HISTORY_STATUS_ADDRESS);
private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY";
@@ -123,7 +122,7 @@ public class HadoopUtils implements Closeable {
try {
configuration = new HdfsConfiguration();
- String resourceStorageType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
+ String resourceStorageType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType);
if (resUploadType == ResUploadType.HDFS){
@@ -156,18 +155,22 @@ public class HadoopUtils implements Closeable {
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS);
}
+ if (fs == null) {
if (StringUtils.isNotEmpty(hdfsUser)) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
- ugi.doAs((PrivilegedExceptionAction<Boolean>) () -> {
+ ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
+ @Override
+ public Boolean run() throws Exception {
fs = FileSystem.get(configuration);
return true;
+ }
});
} else {
logger.warn("hdfs.root.user is not set value!");
fs = FileSystem.get(configuration);
}
+ }
} else if (resUploadType == ResUploadType.S3) {
- System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE);
configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS));
configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT));
configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY));
@@ -201,23 +204,23 @@ public class HadoopUtils implements Closeable {
* if rmHaIds is empty, single resourcemanager enabled
* if rmHaIds not empty: resourcemanager HA enabled
*/
+ String appUrl = "";
+ if (StringUtils.isEmpty(rmHaIds)) {
+ //single resourcemanager enabled
+ appUrl = appAddress;
yarnEnabled = true;
- String appUrl = StringUtils.isEmpty(rmHaIds) ? appAddress : getAppAddress(appAddress, rmHaIds);
- if (StringUtils.isBlank(appUrl)) {
- throw new Exception("yarn application url generation failed");
- }
- if (logger.isDebugEnabled()) {
- logger.debug("yarn application url:{}, applicationId:{}", appUrl, applicationId);
+ } else {
+ //resourcemanager HA enabled
+ appUrl = getAppAddress(appAddress, rmHaIds);
+ yarnEnabled = true;
+ logger.info("application url : {}", appUrl);
}
- String activeResourceManagerPort = String.valueOf(PropertyUtils.getInt(Constants.HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT, 8088));
- return String.format(appUrl, activeResourceManagerPort, applicationId);
- }
- public String getJobHistoryUrl(String applicationId) {
- //eg:application_1587475402360_712719 -> job_1587475402360_712719
- String jobId = applicationId.replace("application", "job");
- return String.format(jobHistoryAddress, jobId);
+ if (StringUtils.isBlank(appUrl)) {
+ throw new Exception("application url is blank");
+ }
+ return String.format(appUrl, applicationId);
}
/**
diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
index b658029..5bb7c39 100644
--- a/dolphinscheduler-common/src/main/resources/common.properties
+++ b/dolphinscheduler-common/src/main/resources/common.properties
@@ -61,9 +61,7 @@ fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
-yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
-# if custom you resourcemanager port ,you need to replace 8088 else default value.
-resource.manager.httpaddress.port=8088
+yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s
# system env path
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh