You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ki...@apache.org on 2021/07/20 06:28:45 UTC
[dolphinscheduler] branch revert-5860-1.3.7-prepare#5256 created
(now a511b1c)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a change to branch revert-5860-1.3.7-prepare#5256
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git.
at a511b1c Revert "[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)"
This branch includes the following new commits:
new a511b1c Revert "[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[dolphinscheduler] 01/01: Revert
"[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)"
Posted by ki...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch revert-5860-1.3.7-prepare#5256
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit a511b1c6ca2096e707671885893a249aa4f5a94e
Author: Kirs <AC...@yeah.net>
AuthorDate: Tue Jul 20 14:28:40 2021 +0800
Revert "[1.3.7-prepare#5256][Improvement][common]Task status error (#5860)"
This reverts commit 8e2c312d4e8e4d9644fda4be6113ca1dd98884d4.
---
.../apache/dolphinscheduler/common/Constants.java | 9 ------
.../dolphinscheduler/common/utils/HadoopUtils.java | 37 ++++++++++++----------
.../src/main/resources/common.properties | 4 +--
3 files changed, 21 insertions(+), 29 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index e8447fa..fe83fa1 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -104,11 +104,6 @@ public final class Constants {
public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address";
/**
- * yarn.job.history.status.address
- */
- public static final String YARN_JOB_HISTORY_STATUS_ADDRESS = "yarn.job.history.status.address";
-
- /**
* hdfs configuration
* hdfs.root.user
*/
@@ -797,10 +792,6 @@ public final class Constants {
*/
public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state";
- /**
- * com.amazonaws.services.s3.enableV4
- */
- public static final String AWS_S3_V4 = "com.amazonaws.services.s3.enableV4";
/**
* loginUserFromKeytab user
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
index 667838d..ffead0b 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
@@ -70,7 +70,6 @@ public class HadoopUtils implements Closeable {
public static final String resourceUploadPath = PropertyUtils.getString(RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
public static final String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS);
public static final String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS);
- public static final String jobHistoryAddress = PropertyUtils.getString(Constants.YARN_JOB_HISTORY_STATUS_ADDRESS);
private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY";
@@ -123,7 +122,7 @@ public class HadoopUtils implements Closeable {
try {
configuration = new HdfsConfiguration();
- String resourceStorageType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
+ String resourceStorageType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType);
if (resUploadType == ResUploadType.HDFS){
@@ -156,18 +155,22 @@ public class HadoopUtils implements Closeable {
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS);
}
+ if (fs == null) {
if (StringUtils.isNotEmpty(hdfsUser)) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
- ugi.doAs((PrivilegedExceptionAction<Boolean>) () -> {
+ ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
+ @Override
+ public Boolean run() throws Exception {
fs = FileSystem.get(configuration);
return true;
+ }
});
} else {
logger.warn("hdfs.root.user is not set value!");
fs = FileSystem.get(configuration);
}
+ }
} else if (resUploadType == ResUploadType.S3) {
- System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE);
configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS));
configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT));
configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY));
@@ -201,23 +204,23 @@ public class HadoopUtils implements Closeable {
* if rmHaIds is empty, single resourcemanager enabled
* if rmHaIds not empty: resourcemanager HA enabled
*/
+ String appUrl = "";
+ if (StringUtils.isEmpty(rmHaIds)) {
+ //single resourcemanager enabled
+ appUrl = appAddress;
yarnEnabled = true;
- String appUrl = StringUtils.isEmpty(rmHaIds) ? appAddress : getAppAddress(appAddress, rmHaIds);
- if (StringUtils.isBlank(appUrl)) {
- throw new Exception("yarn application url generation failed");
- }
- if (logger.isDebugEnabled()) {
- logger.debug("yarn application url:{}, applicationId:{}", appUrl, applicationId);
+ } else {
+ //resourcemanager HA enabled
+ appUrl = getAppAddress(appAddress, rmHaIds);
+ yarnEnabled = true;
+ logger.info("application url : {}", appUrl);
}
- String activeResourceManagerPort = String.valueOf(PropertyUtils.getInt(Constants.HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT, 8088));
- return String.format(appUrl, activeResourceManagerPort, applicationId);
- }
- public String getJobHistoryUrl(String applicationId) {
- //eg:application_1587475402360_712719 -> job_1587475402360_712719
- String jobId = applicationId.replace("application", "job");
- return String.format(jobHistoryAddress, jobId);
+ if (StringUtils.isBlank(appUrl)) {
+ throw new Exception("application url is blank");
+ }
+ return String.format(appUrl, applicationId);
}
/**
diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
index b658029..5bb7c39 100644
--- a/dolphinscheduler-common/src/main/resources/common.properties
+++ b/dolphinscheduler-common/src/main/resources/common.properties
@@ -61,9 +61,7 @@ fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
-yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
-# if custom you resourcemanager port ,you need to replace 8088 else default value.
-resource.manager.httpaddress.port=8088
+yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s
# system env path
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh