You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by jo...@apache.org on 2019/12/12 06:21:07 UTC
[incubator-dolphinscheduler] branch dev updated: Support Mac local
development and debugging (#1435)
This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 6ec5f2e Support Mac local development and debugging (#1435)
6ec5f2e is described below
commit 6ec5f2e3cdbb56528ebf32cdb4aad23952a1d8fb
Author: elonlo <ga...@gmail.com>
AuthorDate: Thu Dec 12 14:20:59 2019 +0800
Support Mac local development and debugging (#1435)
- compatible with pstree command
- compatible with create user command
- compatible with list user command
---
.../dolphinscheduler/common/utils/FileUtils.java | 9 +-
.../dolphinscheduler/common/utils/OSUtils.java | 105 ++++++++++++++++++---
.../dolphinscheduler/common/utils/OSUtilsTest.java | 34 +++++++
.../server/utils/ProcessUtils.java | 17 +++-
.../server/worker/runner/FetchTaskThread.java | 2 +-
.../server/utils/ProcessUtilsTest.java | 33 +++++++
6 files changed, 172 insertions(+), 28 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index c49e5cb..e7da97d 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -127,14 +127,7 @@ public class FileUtils {
//if not exists this user,then create
if (!OSUtils.getUserList().contains(userName)){
- String userGroup = OSUtils.getGroup();
- if (org.apache.commons.lang3.StringUtils.isNotEmpty(userGroup)){
- logger.info("create os user : {}",userName);
- String cmd = String.format("sudo useradd -g %s %s",userGroup,userName);
-
- logger.info("execute cmd : {}",cmd);
- OSUtils.exeCmd(cmd);
- }
+ OSUtils.createUser(userName);
}
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
index fbb3970..20336af 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
@@ -38,6 +38,8 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.DecimalFormat;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
/**
@@ -132,18 +134,32 @@ public class OSUtils {
return Double.parseDouble(df.format(cpuUsage));
}
+ public static List<String> getUserList() {
+ try {
+ if (isMacOS()) {
+ return getUserListFromMac();
+ } else if (isWindows()) {
+ // do something
+ } else {
+ return getUserListFromLinux();
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+
+ return Collections.emptyList();
+ }
/**
- * get user list
+ * get user list from linux
*
* @return user list
*/
- public static List<String> getUserList() {
+ private static List<String> getUserListFromLinux() throws IOException {
List<String> userList = new ArrayList<>();
- BufferedReader bufferedReader = null;
- try {
- bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream("/etc/passwd")));
+ try (BufferedReader bufferedReader = new BufferedReader(
+ new InputStreamReader(new FileInputStream("/etc/passwd")))) {
String line;
while ((line = bufferedReader.readLine()) != null) {
@@ -152,19 +168,80 @@ public class OSUtils {
userList.add(userInfo[0]);
}
}
+ }
+
+ return userList;
+ }
+
+ /**
+ * get user list from mac
+ * @return user list
+ */
+ private static List<String> getUserListFromMac() throws IOException {
+ String result = exeCmd("dscl . list /users");
+ if (StringUtils.isNotEmpty(result)) {
+ return Arrays.asList(StringUtils.split(result, "\n"));
+ }
+
+ return Collections.emptyList();
+ }
+
+ /**
+ * create user
+ * @param userName user name
+ * @return true if creation was successful, otherwise false
+ */
+ public static boolean createUser(String userName) {
+ try {
+ String userGroup = OSUtils.getGroup();
+ if (StringUtils.isEmpty(userGroup)) {
+ logger.error("{} group does not exist for this operating system.", userGroup);
+ return false;
+ }
+ if (isMacOS()) {
+ createMacUser(userName, userGroup);
+ } else if (isWindows()) {
+ // do something
+ } else {
+ createLinuxUser(userName, userGroup);
+ }
+ return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
- } finally {
- try {
- if (bufferedReader != null) {
- bufferedReader.close();
- }
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- }
}
- return userList;
+ return false;
+ }
+
+ /**
+ * create linux user
+ * @param userName user name
+ * @param userGroup user group
+ * @throws IOException in case of an I/O error
+ */
+ private static void createLinuxUser(String userName, String userGroup) throws IOException {
+ logger.info("create linux os user : {}", userName);
+ String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
+
+ logger.info("execute cmd : {}", cmd);
+ OSUtils.exeCmd(cmd);
+ }
+
+ /**
+ * create mac user (Supports Mac OSX 10.10+)
+ * @param userName user name
+ * @param userGroup user group
+ * @throws IOException in case of an I/O error
+ */
+ private static void createMacUser(String userName, String userGroup) throws IOException {
+ logger.info("create mac os user : {}", userName);
+ String userCreateCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
+ String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
+
+ logger.info("create user command : {}", userCreateCmd);
+ OSUtils.exeCmd(userCreateCmd);
+ logger.info("append user to group : {}", appendGroupCmd);
+ OSUtils.exeCmd(appendGroupCmd);
}
/**
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
new file mode 100644
index 0000000..391fb59
--- /dev/null
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.List;
+
+public class OSUtilsTest {
+ private static final Logger logger = LoggerFactory.getLogger(OSUtilsTest.class);
+
+ @Test
+ public void getUserList() {
+ List<String> userList = OSUtils.getUserList();
+ Assert.assertNotEquals("System user list should not be empty", userList.size(), 0);
+ logger.info("OS user list : {}", userList.toString());
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
index a757d62..748c8f0 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
@@ -348,13 +348,20 @@ public class ProcessUtils {
* @return pids
* @throws Exception exception
*/
- private static String getPidsStr(int processId)throws Exception{
+ public static String getPidsStr(int processId)throws Exception{
StringBuilder sb = new StringBuilder();
- // pstree -p pid get sub pids
- String pids = OSUtils.exeCmd("pstree -p " +processId+ "");
- Matcher mat = Pattern.compile("(\\d+)").matcher(pids);
+ Matcher mat;
+ // pstree pid get sub pids
+ if (OSUtils.isMacOS()) {
+ String pids = OSUtils.exeCmd("pstree -sp " + processId);
+ mat = Pattern.compile("-[+|-]-\\s(\\d+)").matcher(pids);
+ } else {
+ String pids = OSUtils.exeCmd("pstree -p " + processId);
+ mat = Pattern.compile("(\\d+)").matcher(pids);
+ }
+
while (mat.find()){
- sb.append(mat.group()+" ");
+ sb.append(mat.group(1)).append(" ");
}
return sb.toString().trim();
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
index f7ea3fa..ae4ee79 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
@@ -222,7 +222,7 @@ public class FetchTaskThread implements Runnable{
new Date(),
execLocalPath);
- // check and create Linux users
+ // check and create users
FileUtils.createWorkDirAndUserIfAbsent(execLocalPath,
tenant.getTenantCode(), logger);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
new file mode 100644
index 0000000..77fc398
--- /dev/null
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProcessUtilsTest {
+ private static final Logger logger = LoggerFactory.getLogger(ProcessUtilsTest.class);
+
+ @Test
+ public void getPidsStr() throws Exception {
+ String pidList = ProcessUtils.getPidsStr(1);
+ Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, "");
+ logger.info("Sub process list : {}", pidList);
+ }
+}