You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2016/03/22 20:45:09 UTC

[1/4] storm git commit: [STORM-1631] - Storm CGroup bug when launching workers as the user that submitted the topology

Repository: storm
Updated Branches:
  refs/heads/master 4e6cb58a9 -> 0b3ee9075


[STORM-1631] - Storm CGroup bug when launching workers as the user that submitted the topology


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f679dac1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f679dac1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f679dac1

Branch: refs/heads/master
Commit: f679dac1733e5c20fabfda34634d0470466eb539
Parents: 500ef20
Author: Boyang Jerry Peng <je...@yahoo-inc.com>
Authored: Tue Mar 15 16:43:58 2016 -0500
Committer: Boyang Jerry Peng <je...@yahoo-inc.com>
Committed: Tue Mar 15 16:43:58 2016 -0500

----------------------------------------------------------------------
 .../clj/org/apache/storm/daemon/supervisor.clj  | 24 +++++++++++++++-----
 .../container/ResourceIsolationInterface.java   |  8 +++++++
 .../storm/container/cgroup/CgroupManager.java   | 16 ++++++++-----
 3 files changed, 36 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/f679dac1/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
index fd8f6c9..6207137 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -241,14 +241,16 @@
 (defn generate-supervisor-id []
   (Utils/uuid))
 
-(defnk worker-launcher [conf user args :environment {} :log-prefix nil :exit-code-callback nil :directory nil]
+(defnk worker-launcher [conf user args :environment {} :log-prefix nil :exit-code-callback nil :directory nil :launch-in-container? false :supervisor nil :worker-id nil]
   (let [_ (when (clojure.string/blank? user)
             (throw (java.lang.IllegalArgumentException.
                      "User cannot be blank when calling worker-launcher.")))
         wl-initial (conf SUPERVISOR-WORKER-LAUNCHER)
         storm-home (System/getProperty "storm.home")
         wl (if wl-initial wl-initial (str storm-home "/bin/worker-launcher"))
-        command (concat [wl user] args)]
+        command (if launch-in-container?
+                  (concat (.getLaunchCommandPrefix (:resource-isolation-manager supervisor) worker-id) [wl user] args)
+                  (concat [wl user] args))]
     (log-message "Running as user:" user " command:" (pr-str command))
     (Utils/launchProcess command
                          environment
@@ -1250,14 +1252,14 @@
           command (->> command
                        (map str)
                        (filter (complement empty?)))
-          command (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
+          command_final (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
                     (do
                       (.reserveResourcesForWorker (:resource-isolation-manager supervisor) worker-id
                         {"cpu" cpu "memory" (+ mem-onheap mem-offheap  (int (Math/ceil (conf STORM-CGROUP-MEMORY-LIMIT-TOLERANCE-MARGIN-MB))))})
                       (.getLaunchCommand (:resource-isolation-manager supervisor) worker-id
                         (java.util.ArrayList. (java.util.Arrays/asList (to-array command)))))
                     command)]
-      (log-message "Launching worker with command: " (Utils/shellCmd command))
+      (log-message "Launching worker with command: " (Utils/shellCmd command_final))
       (write-log-metadata! storm-conf user worker-id storm-id port conf)
       (ConfigUtils/setWorkerUserWSE conf worker-id user)
       (create-artifacts-link conf storm-id port worker-id)
@@ -1270,8 +1272,18 @@
         (remove-dead-worker worker-id)
         (create-blobstore-links conf storm-id worker-id)
         (if run-worker-as-user
-          (worker-launcher conf user ["worker" worker-dir (Utils/writeScript worker-dir command topology-worker-environment)] :log-prefix log-prefix :exit-code-callback callback :directory (File. worker-dir))
-          (Utils/launchProcess command
+          (worker-launcher conf
+            user
+            ["worker"
+             worker-dir
+             (Utils/writeScript worker-dir command topology-worker-environment)]
+            :log-prefix log-prefix
+            :exit-code-callback callback
+            :directory (File. worker-dir)
+            :launch-in-container? (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE) true false)
+            :supervisor supervisor
+            :worker-id worker-id)
+          (Utils/launchProcess command_final
                                topology-worker-environment
                                log-prefix
                                callback

http://git-wip-us.apache.org/repos/asf/storm/blob/f679dac1/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java b/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
index 2db9f1b..c5cad02 100644
--- a/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
+++ b/storm-core/src/jvm/org/apache/storm/container/ResourceIsolationInterface.java
@@ -48,4 +48,12 @@ public interface ResourceIsolationInterface {
      */
     List<String> getLaunchCommand(String workerId, List<String> existingCommand);
 
+    /**
+     * After reserving resources for the worker (i.e. calling reserveResourcesForWorker). this function can be used
+     * to get the launch command prefix
+     * @param workerId the of the worker
+     * @return the command line prefix for launching a worker with resource isolation
+     */
+    List<String> getLaunchCommandPrefix(String workerId);
+
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/f679dac1/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java b/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
index 875474a..80093b3 100644
--- a/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
+++ b/storm-core/src/jvm/org/apache/storm/container/cgroup/CgroupManager.java
@@ -176,12 +176,17 @@ public class CgroupManager implements ResourceIsolationInterface {
 
     @Override
     public List<String> getLaunchCommand(String workerId, List<String> existingCommand) {
+        List<String> newCommand = getLaunchCommandPrefix(workerId);
+        newCommand.addAll(existingCommand);
+        return newCommand;
+    }
 
+    @Override
+    public List<String> getLaunchCommandPrefix(String workerId) {
         CgroupCommon workerGroup = new CgroupCommon(workerId, this.hierarchy, this.rootCgroup);
 
-        if(!this.rootCgroup.getChildren().contains(workerGroup)) {
-            LOG.error("cgroup {} doesn't exist! Need to reserve resources for worker first!", workerGroup);
-            return existingCommand;
+        if (!this.rootCgroup.getChildren().contains(workerGroup)) {
+            throw new RuntimeException("cgroup " + workerGroup + " doesn't exist! Need to reserve resources for worker first!");
         }
 
         StringBuilder sb = new StringBuilder();
@@ -189,9 +194,9 @@ public class CgroupManager implements ResourceIsolationInterface {
         sb.append(this.conf.get(Config.STORM_CGROUP_CGEXEC_CMD)).append(" -g ");
 
         Iterator<SubSystemType> it = this.hierarchy.getSubSystems().iterator();
-        while(it.hasNext()) {
+        while (it.hasNext()) {
             sb.append(it.next().toString());
-            if(it.hasNext()) {
+            if (it.hasNext()) {
                 sb.append(",");
             } else {
                 sb.append(":");
@@ -200,7 +205,6 @@ public class CgroupManager implements ResourceIsolationInterface {
         sb.append(workerGroup.getName());
         List<String> newCommand = new ArrayList<String>();
         newCommand.addAll(Arrays.asList(sb.toString().split(" ")));
-        newCommand.addAll(existingCommand);
         return newCommand;
     }
 


[4/4] storm git commit: Added STORM-1631 to Changelog

Posted by bo...@apache.org.
Added STORM-1631 to Changelog


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0b3ee907
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0b3ee907
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0b3ee907

Branch: refs/heads/master
Commit: 0b3ee9075b1120dfdfaeb77d269420ead7ca2056
Parents: 4ad21bd
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Mar 22 14:34:12 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Mar 22 14:34:12 2016 -0500

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/0b3ee907/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ae460c9..2c0e1c1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 2.0.0
+ * STORM-1631: Storm CGroup bugs
  * STORM-1616: Add RAS API for Trident
  * STORM-1623: nimbus.clj's minor bug
  * STORM-1624: Add maven central status in README


[3/4] storm git commit: Merge branch 'STORM-1631' of https://github.com/jerrypeng/storm into STORM-1631

Posted by bo...@apache.org.
Merge branch 'STORM-1631' of https://github.com/jerrypeng/storm into STORM-1631

STORM-1631: Storm CGroup bugs


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4ad21bd6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4ad21bd6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4ad21bd6

Branch: refs/heads/master
Commit: 4ad21bd64650cd163efbdc38bc1883219b8ef11f
Parents: 4e6cb58 3b4a4e3
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Mar 22 14:33:35 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Mar 22 14:33:35 2016 -0500

----------------------------------------------------------------------
 .../clj/org/apache/storm/daemon/supervisor.clj  | 31 +++++++++++++++-----
 .../container/ResourceIsolationInterface.java   |  8 +++++
 .../storm/container/cgroup/CgroupManager.java   | 16 ++++++----
 3 files changed, 41 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/4ad21bd6/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------


[2/4] storm git commit: moving cgroups cleanup code to a better place so that retry will happen if failure occurs at cleanup

Posted by bo...@apache.org.
moving cgroups cleanup code to a better place so that retry will happen if failure occurs at cleanup


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3b4a4e3b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3b4a4e3b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3b4a4e3b

Branch: refs/heads/master
Commit: 3b4a4e3b8ea2fcaf0c1d58fb3abcec2966752369
Parents: f679dac
Author: Boyang Jerry Peng <je...@yahoo-inc.com>
Authored: Tue Mar 15 23:42:27 2016 -0500
Committer: Boyang Jerry Peng <je...@yahoo-inc.com>
Committed: Tue Mar 15 23:42:27 2016 -0500

----------------------------------------------------------------------
 storm-core/src/clj/org/apache/storm/daemon/supervisor.clj | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/3b4a4e3b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
index 6207137..d15a9d7 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -282,6 +282,11 @@
 
 (defn try-cleanup-worker [conf supervisor id]
   (try
+    ;; clean up for resource isolation if enabled
+    (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
+      (.releaseResourcesForWorker (:resource-isolation-manager supervisor) id))
+    ;; Always make sure to clean up everything else before worker directory
+    ;; is removed since that is what is going to trigger the retry for cleanup
     (if (.exists (File. (ConfigUtils/workerRoot conf id)))
       (do
         (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
@@ -295,8 +300,6 @@
         (ConfigUtils/removeWorkerUserWSE conf id)
         (remove-dead-worker id)
       ))
-    (if (conf STORM-RESOURCE-ISOLATION-PLUGIN-ENABLE)
-      (.releaseResourcesForWorker (:resource-isolation-manager supervisor) id))
   (catch IOException e
     (log-warn-error e "Failed to cleanup worker " id ". Will retry later"))
   (catch RuntimeException e