You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2016/03/22 20:45:09 UTC
[1/4] storm git commit: [STORM-1631] - Storm CGroup bug when
launching workers as the user that submitted the topology
Repository: storm
Updated Branches:
refs/heads/master 4e6cb58a9 -> 0b3ee9075
[STORM-1631] - Storm CGroup bug when launching workers as the user that submitted the topology
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f679dac1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f679dac1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f679dac1
Branch: refs/heads/master
Commit: f679dac1733e5c20fabfda34634d0470466eb539
Parents: 500ef20
Author: Boyang Jerry Peng <je...@yahoo-inc.com>
Authored: Tue Mar 15 16:43:58 2016 -0500
Committer: Boyang Jerry Peng <je...@yahoo-inc.com>
Committed: Tue Mar 15 16:43:58 2016 -0500
----------------------------------------------------------------------
.../clj/org/apache/storm/daemon/supervisor.clj | 24 +++++++++++++++-----
.../container/ResourceIsolationInterface.java | 8 +++++++
.../storm/container/cgroup/CgroupManager.java | 16 ++++++++-----
3 files changed, 36 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f679dac1/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
index fd8f6c9..6207137 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -241,14 +241,16 @@
(defn generate-supervisor-id []
(Utils/uuid))
-(defnk worker-launcher [conf user args :environment {} :log-prefix nil :exit-code-callback nil :directory nil]
+(defnk worker-launcher [conf user args :environment {} :log-prefix nil :exit-code-callback nil :directory nil :launch-in-container? false :supervisor nil :worker-id nil]
(let [_ (when (clojure.string/blank? user)
(throw (java.lang.IllegalArgumentException.
"User cannot be blank when calling worker-launcher.")))
wl-initial (conf SUPERVISOR-WORKER-LAUNCHER)
storm-home (System/getProperty "storm.home")
wl (if wl-initial wl-initial (str storm-home "/bin/worker-launcher"))
- command (concat [wl user] args)]
+ command (if launch-in-container?
+ (concat (.getLaunchCommandPrefix (:resource-isolation-manager supervisor) worker-id) [wl user] args)
+ (concat [wl user] args))]
(log-message "Running as user:" user " command:" (pr-str command))
(Utils/launchProcess command
environment
@@ -1250,14 +1252,14 @@
command (->> command
(map str)
(filter (complement empty?)))
- command (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
+ command_final (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
(do
(.reserveResourcesForWorker (:resource-isolation-manager supervisor) worker-id
{"cpu" cpu "memory" (+ mem-onheap mem-offheap (int (Math/ceil (conf STORM-CGROUP-MEMORY-LIMIT-TOLERANCE-MARGIN-MB))))})
(.getLaunchCommand (:resource-isolation-manager supervisor) worker-id
(java.util.ArrayList. (java.util.Arrays/asList (to-array command)))))
command)]
- (log-message "Launching worker with command: " (Utils/shellCmd command))
+ (log-message "Launching worker with command: " (Utils/shellCmd command_final))
(write-log-metadata! storm-conf user worker-id storm-id port conf)
(ConfigUtils/setWorkerUserWSE conf worker-id user)
(create-artifacts-link conf storm-id port worker-id)
@@ -1270,8 +1272,18 @@
(remove-dead-worker worker-id)
(create-blobstore-links conf storm-id worker-id)
(if run-worker-as-user
- (worker-launcher conf user ["worker" worker-dir (Utils/writeScript worker-dir command topology-worker-environment)] :log-prefix log-prefix :exit-code-callback callback :directory (File. worker-dir))
- (Utils/launchProcess command
+ (worker-launcher conf
+ user
+ ["worker"
+ worker-dir
+ (Utils/writeScript worker-dir command topology-worker-environment)]
+ :log-prefix log-prefix
+ :exit-code-callback callback
+ :directory (File. worker-dir)
+ :launch-in-container? (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE) true false)
+ :supervisor supervisor
+ :worker-id worker-id)
+ (Utils/launchProcess command_final
topology-worker-environment
log-prefix
callback
http://git-wip-us.apache.org/repos/asf/storm/blob/f679dac1/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java b/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
index 2db9f1b..c5cad02 100644
--- a/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
+++ b/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
@@ -48,4 +48,12 @@ public interface ResourceIsolationInterface {
*/
List<String> getLaunchCommand(String workerId, List<String> existingCommand);
+ /**
+ * After reserving resources for the worker (i.e. calling reserveResourcesForWorker). this function can be used
+ * to get the launch command prefix
+ * @param workerId the of the worker
+ * @return the command line prefix for launching a worker with resource isolation
+ */
+ List<String> getLaunchCommandPrefix(String workerId);
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f679dac1/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java b/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
index 875474a..80093b3 100644
--- a/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
+++ b/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
@@ -176,12 +176,17 @@ public class CgroupManager implements ResourceIsolationInterface {
@Override
public List<String> getLaunchCommand(String workerId, List<String> existingCommand) {
+ List<String> newCommand = getLaunchCommandPrefix(workerId);
+ newCommand.addAll(existingCommand);
+ return newCommand;
+ }
+ @Override
+ public List<String> getLaunchCommandPrefix(String workerId) {
CgroupCommon workerGroup = new CgroupCommon(workerId, this.hierarchy, this.rootCgroup);
- if(!this.rootCgroup.getChildren().contains(workerGroup)) {
- LOG.error("cgroup {} doesn't exist! Need to reserve resources for worker first!", workerGroup);
- return existingCommand;
+ if (!this.rootCgroup.getChildren().contains(workerGroup)) {
+ throw new RuntimeException("cgroup " + workerGroup + " doesn't exist! Need to reserve resources for worker first!");
}
StringBuilder sb = new StringBuilder();
@@ -189,9 +194,9 @@ public class CgroupManager implements ResourceIsolationInterface {
sb.append(this.conf.get(Config.STORM_CGROUP_CGEXEC_CMD)).append(" -g ");
Iterator<SubSystemType> it = this.hierarchy.getSubSystems().iterator();
- while(it.hasNext()) {
+ while (it.hasNext()) {
sb.append(it.next().toString());
- if(it.hasNext()) {
+ if (it.hasNext()) {
sb.append(",");
} else {
sb.append(":");
@@ -200,7 +205,6 @@ public class CgroupManager implements ResourceIsolationInterface {
sb.append(workerGroup.getName());
List<String> newCommand = new ArrayList<String>();
newCommand.addAll(Arrays.asList(sb.toString().split(" ")));
- newCommand.addAll(existingCommand);
return newCommand;
}
[4/4] storm git commit: Added STORM-1631 to Changelog
Posted by bo...@apache.org.
Added STORM-1631 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0b3ee907
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0b3ee907
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0b3ee907
Branch: refs/heads/master
Commit: 0b3ee9075b1120dfdfaeb77d269420ead7ca2056
Parents: 4ad21bd
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Mar 22 14:34:12 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Mar 22 14:34:12 2016 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0b3ee907/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ae460c9..2c0e1c1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-1631: Storm CGroup bugs
* STORM-1616: Add RAS API for Trident
* STORM-1623: nimbus.clj's minor bug
* STORM-1624: Add maven central status in README
[3/4] storm git commit: Merge branch 'STORM-1631' of
https://github.com/jerrypeng/storm into STORM-1631
Posted by bo...@apache.org.
Merge branch 'STORM-1631' of https://github.com/jerrypeng/storm into STORM-1631
STORM-1631: Storm CGroup bugs
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4ad21bd6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4ad21bd6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4ad21bd6
Branch: refs/heads/master
Commit: 4ad21bd64650cd163efbdc38bc1883219b8ef11f
Parents: 4e6cb58 3b4a4e3
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Mar 22 14:33:35 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Mar 22 14:33:35 2016 -0500
----------------------------------------------------------------------
.../clj/org/apache/storm/daemon/supervisor.clj | 31 +++++++++++++++-----
.../container/ResourceIsolationInterface.java | 8 +++++
.../storm/container/cgroup/CgroupManager.java | 16 ++++++----
3 files changed, 41 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4ad21bd6/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
[2/4] storm git commit: moving cgroups cleanup code to a better place
so that retry will happen if failure occurs at cleanup
Posted by bo...@apache.org.
moving cgroups cleanup code to a better place so that retry will happen if failure occurs at cleanup
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3b4a4e3b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3b4a4e3b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3b4a4e3b
Branch: refs/heads/master
Commit: 3b4a4e3b8ea2fcaf0c1d58fb3abcec2966752369
Parents: f679dac
Author: Boyang Jerry Peng <je...@yahoo-inc.com>
Authored: Tue Mar 15 23:42:27 2016 -0500
Committer: Boyang Jerry Peng <je...@yahoo-inc.com>
Committed: Tue Mar 15 23:42:27 2016 -0500
----------------------------------------------------------------------
storm-core/src/clj/org/apache/storm/daemon/supervisor.clj | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3b4a4e3b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
index 6207137..d15a9d7 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -282,6 +282,11 @@
(defn try-cleanup-worker [conf supervisor id]
(try
+ ;; clean up for resource isolation if enabled
+ (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
+ (.releaseResourcesForWorker (:resource-isolation-manager supervisor) id))
+ ;; Always make sure to clean up everything else before worker directory
+ ;; is removed since that is what is going to trigger the retry for cleanup
(if (.exists (File. (ConfigUtils/workerRoot conf id)))
(do
(if (conf SUPERVISOR-RUN-WORKER-AS-USER)
@@ -295,8 +300,6 @@
(ConfigUtils/removeWorkerUserWSE conf id)
(remove-dead-worker id)
))
- (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
- (.releaseResourcesForWorker (:resource-isolation-manager supervisor) id))
(catch IOException e
(log-warn-error e "Failed to cleanup worker " id ". Will retry later"))
(catch RuntimeException e