You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by et...@apache.org on 2018/04/30 03:22:43 UTC

[1/2] storm git commit: [STORM-3021] Fix wrong usages of Config.TOPOLOGY_WORKERS on RAS cluster

Repository: storm
Updated Branches:
  refs/heads/master e29476177 -> e62ee2852


[STORM-3021] Fix wrong usages of Config.TOPOLOGY_WORKERS on RAS cluster


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f9734727
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f9734727
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f9734727

Branch: refs/heads/master
Commit: f97347277ba2ad237c0d72ee80334fbf45bbe62f
Parents: 4137328
Author: Ethan Li <et...@gmail.com>
Authored: Thu Apr 12 22:10:25 2018 -0500
Committer: Ethan Li <et...@gmail.com>
Committed: Fri Apr 27 09:55:12 2018 -0500

----------------------------------------------------------------------
 docs/Resource_Aware_Scheduler_overview.md       |  4 ++
 .../src/jvm/org/apache/storm/Config.java        | 15 +++---
 .../org/apache/storm/daemon/worker/Worker.java  |  9 ++--
 .../apache/storm/daemon/worker/WorkerState.java | 19 +++++++
 .../java/org/apache/storm/DaemonConfig.java     |  1 +
 .../org/apache/storm/daemon/nimbus/Nimbus.java  | 55 +++++++++-----------
 6 files changed, 61 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/docs/Resource_Aware_Scheduler_overview.md
----------------------------------------------------------------------
diff --git a/docs/Resource_Aware_Scheduler_overview.md b/docs/Resource_Aware_Scheduler_overview.md
index 335b467..4d03f88 100644
--- a/docs/Resource_Aware_Scheduler_overview.md
+++ b/docs/Resource_Aware_Scheduler_overview.md
@@ -184,6 +184,10 @@ The user can set some default configurations for the Resource Aware Scheduler in
     topology.worker.max.heap.size.mb: 768.0
 ```
 
+### Warning
+
+If Resource Aware Scheduling is enabled, it will dynamically calculate the number of workers and the `topology.workers` setting is ignored.
+
 <div id='Topology-Priorities-and-Per-User-Resource'/>
 
 ## Topology Priorities and Per User Resource 

http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-client/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index d8b0811..61b4518 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -180,7 +180,8 @@ public class Config extends HashMap<String, Object> {
     /**
      * How many processes should be spawned around the cluster to execute this topology. Each process will execute some number of tasks as
      * threads within them. This parameter should be used in conjunction with the parallelism hints on each component in the topology to
-     * tune the performance of a topology.
+     * tune the performance of a topology. The number of workers will be dynamically calculated when the Resource Aware scheduler is used,
+     * in which case this parameter will not be honored.
      */
     @isInteger
     @isPositiveNumber
@@ -331,9 +332,10 @@ public class Config extends HashMap<String, Object> {
     /**
      * How many executors to spawn for ackers.
      *
-     * <p>By not setting this variable or setting it as null, Storm will set the number of acker executors
-     * to be equal to the number of workers configured for this topology. If this variable is set to 0, then Storm will immediately ack
-     * tuples as soon as they come off the spout, effectively disabling reliability.</p>
+     * <p>By not setting this variable or setting it as null, Storm will set the number of acker executors to be equal to
+     * the number of workers configured for this topology (or the estimated number of workers if the Resource Aware Scheduler is used).
+     * If this variable is set to 0, then Storm will immediately ack tuples as soon as they come off the spout,
+     * effectively disabling reliability.</p>
      */
     @isInteger
     @isPositiveNumber(includeZero = true)
@@ -351,8 +353,9 @@ public class Config extends HashMap<String, Object> {
     /**
      * How many executors to spawn for event logger.
      *
-     * <p>By not setting this variable or setting it as null, Storm will set the number of eventlogger executors
-     * to be equal to the number of workers configured for this topology. If this variable is set to 0, event logging will be disabled.</p>
+     * <p>By setting it as null, Storm will set the number of eventlogger executors to be equal to the number of workers
+     * configured for this topology (or the estimated number of workers if the Resource Aware Scheduler is used).
+     * If this variable is set to 0, event logging will be disabled.</p>
      */
     @isInteger
     @isPositiveNumber(includeZero = true)

http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
index a050b06..3a89d0e 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
@@ -231,10 +231,8 @@ public class Worker implements Shutdownable, DaemonCommon {
         executorsAtom.set(newExecutors);
 
         // This thread will send out messages destined for remote tasks (on other workers)
-        // If there are no remote tasks, don't start the thread.
-        Set<Integer> remoteTasks = Sets.difference(new HashSet<>(workerState.getOutboundTasks()),
-                                                   new HashSet<>(workerState.getLocalTaskIds()));
-        if (!remoteTasks.isEmpty()) {
+        // If there are no remote outbound tasks, don't start the thread.
+        if (workerState.hasRemoteOutboundTasks()) {
             transferThread = workerState.makeTransferThread();
             transferThread.setName("Worker-Transfer");
         }
@@ -313,8 +311,7 @@ public class Worker implements Shutdownable, DaemonCommon {
     }
 
     private void setupBackPressureCheckTimer(final Map<String, Object> topologyConf) {
-        final Integer workerCount = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_WORKERS));
-        if (workerCount <= 1) {
+        if (workerState.isSingleWorker()) {
             LOG.info("BackPressure change checking is disabled as there is only one worker");
             return;
         }

http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
index 30cea01..fe6355b 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
@@ -731,6 +731,25 @@ public class WorkerState {
         return this.outboundTasks;
     }
 
+    /**
+     * Check if this worker has remote outbound tasks.
+     * @return true if this worker has remote outbound tasks; false otherwise.
+     */
+    public boolean hasRemoteOutboundTasks() {
+        Set<Integer> remoteTasks = Sets.difference(new HashSet<>(outboundTasks), new HashSet<>(localTaskIds));
+        return !remoteTasks.isEmpty();
+    }
+
+    /**
+     * If all the tasks are local tasks, the topology has only one worker.
+     * @return true if this worker is the single worker; false otherwise.
+     */
+    public boolean isSingleWorker() {
+        Set<Integer> nonLocalTasks = Sets.difference(getTaskToComponent().keySet(),
+                new HashSet<>(localTaskIds));
+        return nonLocalTasks.isEmpty();
+    }
+
     public void haltWorkerTransfer() {
         workerTransfer.haltTransferThd();
     }

http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
index 8fda533..2487414 100644
--- a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
+++ b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
@@ -684,6 +684,7 @@ public class DaemonConfig implements Validated {
 
     /**
      * A number representing the maximum number of workers any single topology can acquire.
+     * This will be ignored if the Resource Aware Scheduler is used.
      */
     @isInteger
     @isPositiveNumber(includeZero = true)

http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
index 6d40db9..d18553f 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
@@ -968,22 +968,24 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
     }
 
     private static void validateTopologySize(Map<String, Object> topoConf, Map<String, Object> nimbusConf,
-                                             StormTopology topology) throws InvalidTopologyException {
-        int workerCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 1);
-        Integer allowedWorkers = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_SLOTS_PER_TOPOLOGY), null);
+        StormTopology topology) throws InvalidTopologyException {
+        // check allowedWorkers only if the scheduler is not the Resource Aware Scheduler
+        if (!ServerUtils.isRAS(nimbusConf)) {
+            int workerCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 1);
+            Integer allowedWorkers = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_SLOTS_PER_TOPOLOGY), null);
+            if (allowedWorkers != null && workerCount > allowedWorkers) {
+                throw new InvalidTopologyException("Failed to submit topology. Topology requests more than "
+                        + allowedWorkers + " workers.");
+            }
+        }
         int executorsCount = 0;
         for (Object comp : StormCommon.allComponents(topology).values()) {
             executorsCount += StormCommon.numStartExecutors(comp);
         }
         Integer allowedExecutors = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_EXECUTORS_PER_TOPOLOGY), null);
         if (allowedExecutors != null && executorsCount > allowedExecutors) {
-            throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " +
-                                               allowedExecutors + " executors.");
-        }
-
-        if (allowedWorkers != null && workerCount > allowedWorkers) {
-            throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " +
-                                               allowedWorkers + " workers.");
+            throw new InvalidTopologyException("Failed to submit topology. Topology requests more than "
+                    + allowedExecutors + " executors.");
         }
     }
 
@@ -2860,23 +2862,6 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
         submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE));
     }
 
-    //get the number of acker executors. We need to estimate it if it's on RAS cluster.
-    private int getNumOfAckerExecs(Map<String, Object> totalConf, StormTopology topology) throws InvalidTopologyException {
-        Object ackerNum = totalConf.get(Config.TOPOLOGY_ACKER_EXECUTORS);
-        if (ackerNum != null) {
-            return ObjectReader.getInt(ackerNum);
-        } else {
-            // if it's resource aware scheduler, estimates the number of acker executors.
-            if (ServerUtils.isRAS(totalConf)) {
-                return ServerUtils.getEstimatedWorkerCountForRASTopo(totalConf, topology);
-            } else {
-                return ObjectReader.getInt(totalConf.get(Config.TOPOLOGY_WORKERS));
-            }
-        }
-    }
-
-    //THRIFT SERVER METHODS...
-
     private void upsertWorkerTokensInCreds(Map<String, String> creds, String user, String topologyId) {
         if (workerTokenManager != null) {
             final long renewIfExpirationBefore = workerTokenManager.getMaxExpirationTimeForRenewal();
@@ -2987,9 +2972,19 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
             // if the other config does not have it set.
             topology = normalizeTopology(totalConf, topology);
 
-            //set the number of acker executors;
-            totalConfToSave.put(Config.TOPOLOGY_ACKER_EXECUTORS, getNumOfAckerExecs(totalConf, topology));
-            LOG.debug("Config.TOPOLOGY_ACKER_EXECUTORS set to: {}", totalConfToSave.get(Config.TOPOLOGY_ACKER_EXECUTORS));
+            // if the Resource Aware Scheduler is used,
+            // we might need to set the number of acker executors and eventlogger executors to be the estimated number of workers.
+            if (ServerUtils.isRAS(conf)) {
+                int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRASTopo(totalConf, topology);
+                int numAckerExecs = ObjectReader.getInt(totalConf.get(Config.TOPOLOGY_ACKER_EXECUTORS), estimatedNumWorker);
+                int numEventLoggerExecs = ObjectReader.getInt(totalConf.get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS), estimatedNumWorker);
+
+                totalConfToSave.put(Config.TOPOLOGY_ACKER_EXECUTORS, numAckerExecs);
+                totalConfToSave.put(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, numEventLoggerExecs);
+
+                LOG.debug("{} set to: {}", Config.TOPOLOGY_ACKER_EXECUTORS, numAckerExecs);
+                LOG.debug("{} set to: {}", Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, numEventLoggerExecs);
+            }
 
             IStormClusterState state = stormClusterState;
 


[2/2] storm git commit: Merge branch 'STORM-3021' of https://github.com/Ethanlm/storm into STORM-3021

Posted by et...@apache.org.
Merge branch 'STORM-3021' of https://github.com/Ethanlm/storm into STORM-3021


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e62ee285
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e62ee285
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e62ee285

Branch: refs/heads/master
Commit: e62ee285265248009c984218ac01aae0473a04c6
Parents: e294761 f973472
Author: Ethan Li <et...@gmail.com>
Authored: Sun Apr 29 22:22:06 2018 -0500
Committer: Ethan Li <et...@gmail.com>
Committed: Sun Apr 29 22:22:06 2018 -0500

----------------------------------------------------------------------
 docs/Resource_Aware_Scheduler_overview.md       |  4 ++
 .../src/jvm/org/apache/storm/Config.java        | 15 +++---
 .../org/apache/storm/daemon/worker/Worker.java  |  9 ++--
 .../apache/storm/daemon/worker/WorkerState.java | 19 +++++++
 .../java/org/apache/storm/DaemonConfig.java     |  1 +
 .../org/apache/storm/daemon/nimbus/Nimbus.java  | 55 +++++++++-----------
 6 files changed, 61 insertions(+), 42 deletions(-)
----------------------------------------------------------------------