You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by et...@apache.org on 2018/04/30 03:22:43 UTC
[1/2] storm git commit: [STORM-3021] Fix wrong usages of
Config.TOPOLOGY_WORKERS on RAS cluster
Repository: storm
Updated Branches:
refs/heads/master e29476177 -> e62ee2852
[STORM-3021] Fix wrong usages of Config.TOPOLOGY_WORKERS on RAS cluster
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f9734727
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f9734727
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f9734727
Branch: refs/heads/master
Commit: f97347277ba2ad237c0d72ee80334fbf45bbe62f
Parents: 4137328
Author: Ethan Li <et...@gmail.com>
Authored: Thu Apr 12 22:10:25 2018 -0500
Committer: Ethan Li <et...@gmail.com>
Committed: Fri Apr 27 09:55:12 2018 -0500
----------------------------------------------------------------------
docs/Resource_Aware_Scheduler_overview.md | 4 ++
.../src/jvm/org/apache/storm/Config.java | 15 +++---
.../org/apache/storm/daemon/worker/Worker.java | 9 ++--
.../apache/storm/daemon/worker/WorkerState.java | 19 +++++++
.../java/org/apache/storm/DaemonConfig.java | 1 +
.../org/apache/storm/daemon/nimbus/Nimbus.java | 55 +++++++++-----------
6 files changed, 61 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/docs/Resource_Aware_Scheduler_overview.md
----------------------------------------------------------------------
diff --git a/docs/Resource_Aware_Scheduler_overview.md b/docs/Resource_Aware_Scheduler_overview.md
index 335b467..4d03f88 100644
--- a/docs/Resource_Aware_Scheduler_overview.md
+++ b/docs/Resource_Aware_Scheduler_overview.md
@@ -184,6 +184,10 @@ The user can set some default configurations for the Resource Aware Scheduler in
topology.worker.max.heap.size.mb: 768.0
```
+### Warning
+
+If Resource Aware Scheduling is enabled, it will dynamically calculate the number of workers and the `topology.workers` setting is ignored.
+
<div id='Topology-Priorities-and-Per-User-Resource'/>
## Topology Priorities and Per User Resource
http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-client/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index d8b0811..61b4518 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -180,7 +180,8 @@ public class Config extends HashMap<String, Object> {
/**
* How many processes should be spawned around the cluster to execute this topology. Each process will execute some number of tasks as
* threads within them. This parameter should be used in conjunction with the parallelism hints on each component in the topology to
- * tune the performance of a topology.
+ * tune the performance of a topology. The number of workers will be dynamically calculated when the Resource Aware scheduler is used,
+ * in which case this parameter will not be honored.
*/
@isInteger
@isPositiveNumber
@@ -331,9 +332,10 @@ public class Config extends HashMap<String, Object> {
/**
* How many executors to spawn for ackers.
*
- * <p>By not setting this variable or setting it as null, Storm will set the number of acker executors
- * to be equal to the number of workers configured for this topology. If this variable is set to 0, then Storm will immediately ack
- * tuples as soon as they come off the spout, effectively disabling reliability.</p>
+ * <p>By not setting this variable or setting it as null, Storm will set the number of acker executors to be equal to
+ * the number of workers configured for this topology (or the estimated number of workers if the Resource Aware Scheduler is used).
+ * If this variable is set to 0, then Storm will immediately ack tuples as soon as they come off the spout,
+ * effectively disabling reliability.</p>
*/
@isInteger
@isPositiveNumber(includeZero = true)
@@ -351,8 +353,9 @@ public class Config extends HashMap<String, Object> {
/**
* How many executors to spawn for event logger.
*
- * <p>By not setting this variable or setting it as null, Storm will set the number of eventlogger executors
- * to be equal to the number of workers configured for this topology. If this variable is set to 0, event logging will be disabled.</p>
+ * <p>By setting it as null, Storm will set the number of eventlogger executors to be equal to the number of workers
+ * configured for this topology (or the estimated number of workers if the Resource Aware Scheduler is used).
+ * If this variable is set to 0, event logging will be disabled.</p>
*/
@isInteger
@isPositiveNumber(includeZero = true)
http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
index a050b06..3a89d0e 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
@@ -231,10 +231,8 @@ public class Worker implements Shutdownable, DaemonCommon {
executorsAtom.set(newExecutors);
// This thread will send out messages destined for remote tasks (on other workers)
- // If there are no remote tasks, don't start the thread.
- Set<Integer> remoteTasks = Sets.difference(new HashSet<>(workerState.getOutboundTasks()),
- new HashSet<>(workerState.getLocalTaskIds()));
- if (!remoteTasks.isEmpty()) {
+ // If there are no remote outbound tasks, don't start the thread.
+ if (workerState.hasRemoteOutboundTasks()) {
transferThread = workerState.makeTransferThread();
transferThread.setName("Worker-Transfer");
}
@@ -313,8 +311,7 @@ public class Worker implements Shutdownable, DaemonCommon {
}
private void setupBackPressureCheckTimer(final Map<String, Object> topologyConf) {
- final Integer workerCount = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_WORKERS));
- if (workerCount <= 1) {
+ if (workerState.isSingleWorker()) {
LOG.info("BackPressure change checking is disabled as there is only one worker");
return;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
index 30cea01..fe6355b 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
@@ -731,6 +731,25 @@ public class WorkerState {
return this.outboundTasks;
}
+ /**
+ * Check if this worker has remote outbound tasks.
+ * @return true if this worker has remote outbound tasks; false otherwise.
+ */
+ public boolean hasRemoteOutboundTasks() {
+ Set<Integer> remoteTasks = Sets.difference(new HashSet<>(outboundTasks), new HashSet<>(localTaskIds));
+ return !remoteTasks.isEmpty();
+ }
+
+ /**
+ * If all the tasks are local tasks, the topology has only one worker.
+ * @return true if this worker is the single worker; false otherwise.
+ */
+ public boolean isSingleWorker() {
+ Set<Integer> nonLocalTasks = Sets.difference(getTaskToComponent().keySet(),
+ new HashSet<>(localTaskIds));
+ return nonLocalTasks.isEmpty();
+ }
+
public void haltWorkerTransfer() {
workerTransfer.haltTransferThd();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
index 8fda533..2487414 100644
--- a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
+++ b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
@@ -684,6 +684,7 @@ public class DaemonConfig implements Validated {
/**
* A number representing the maximum number of workers any single topology can acquire.
+ * This will be ignored if the Resource Aware Scheduler is used.
*/
@isInteger
@isPositiveNumber(includeZero = true)
http://git-wip-us.apache.org/repos/asf/storm/blob/f9734727/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
index 6d40db9..d18553f 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
@@ -968,22 +968,24 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
private static void validateTopologySize(Map<String, Object> topoConf, Map<String, Object> nimbusConf,
- StormTopology topology) throws InvalidTopologyException {
- int workerCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 1);
- Integer allowedWorkers = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_SLOTS_PER_TOPOLOGY), null);
+ StormTopology topology) throws InvalidTopologyException {
+ // check allowedWorkers only if the scheduler is not the Resource Aware Scheduler
+ if (!ServerUtils.isRAS(nimbusConf)) {
+ int workerCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 1);
+ Integer allowedWorkers = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_SLOTS_PER_TOPOLOGY), null);
+ if (allowedWorkers != null && workerCount > allowedWorkers) {
+ throw new InvalidTopologyException("Failed to submit topology. Topology requests more than "
+ + allowedWorkers + " workers.");
+ }
+ }
int executorsCount = 0;
for (Object comp : StormCommon.allComponents(topology).values()) {
executorsCount += StormCommon.numStartExecutors(comp);
}
Integer allowedExecutors = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_EXECUTORS_PER_TOPOLOGY), null);
if (allowedExecutors != null && executorsCount > allowedExecutors) {
- throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " +
- allowedExecutors + " executors.");
- }
-
- if (allowedWorkers != null && workerCount > allowedWorkers) {
- throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " +
- allowedWorkers + " workers.");
+ throw new InvalidTopologyException("Failed to submit topology. Topology requests more than "
+ + allowedExecutors + " executors.");
}
}
@@ -2860,23 +2862,6 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE));
}
- //get the number of acker executors. We need to estimate it if it's on RAS cluster.
- private int getNumOfAckerExecs(Map<String, Object> totalConf, StormTopology topology) throws InvalidTopologyException {
- Object ackerNum = totalConf.get(Config.TOPOLOGY_ACKER_EXECUTORS);
- if (ackerNum != null) {
- return ObjectReader.getInt(ackerNum);
- } else {
- // if it's resource aware scheduler, estimates the number of acker executors.
- if (ServerUtils.isRAS(totalConf)) {
- return ServerUtils.getEstimatedWorkerCountForRASTopo(totalConf, topology);
- } else {
- return ObjectReader.getInt(totalConf.get(Config.TOPOLOGY_WORKERS));
- }
- }
- }
-
- //THRIFT SERVER METHODS...
-
private void upsertWorkerTokensInCreds(Map<String, String> creds, String user, String topologyId) {
if (workerTokenManager != null) {
final long renewIfExpirationBefore = workerTokenManager.getMaxExpirationTimeForRenewal();
@@ -2987,9 +2972,19 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
// if the other config does not have it set.
topology = normalizeTopology(totalConf, topology);
- //set the number of acker executors;
- totalConfToSave.put(Config.TOPOLOGY_ACKER_EXECUTORS, getNumOfAckerExecs(totalConf, topology));
- LOG.debug("Config.TOPOLOGY_ACKER_EXECUTORS set to: {}", totalConfToSave.get(Config.TOPOLOGY_ACKER_EXECUTORS));
+ // if the Resource Aware Scheduler is used,
+ // we might need to set the number of acker executors and eventlogger executors to be the estimated number of workers.
+ if (ServerUtils.isRAS(conf)) {
+ int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRASTopo(totalConf, topology);
+ int numAckerExecs = ObjectReader.getInt(totalConf.get(Config.TOPOLOGY_ACKER_EXECUTORS), estimatedNumWorker);
+ int numEventLoggerExecs = ObjectReader.getInt(totalConf.get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS), estimatedNumWorker);
+
+ totalConfToSave.put(Config.TOPOLOGY_ACKER_EXECUTORS, numAckerExecs);
+ totalConfToSave.put(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, numEventLoggerExecs);
+
+ LOG.debug("{} set to: {}", Config.TOPOLOGY_ACKER_EXECUTORS, numAckerExecs);
+ LOG.debug("{} set to: {}", Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, numEventLoggerExecs);
+ }
IStormClusterState state = stormClusterState;
[2/2] storm git commit: Merge branch 'STORM-3021' of
https://github.com/Ethanlm/storm into STORM-3021
Posted by et...@apache.org.
Merge branch 'STORM-3021' of https://github.com/Ethanlm/storm into STORM-3021
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e62ee285
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e62ee285
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e62ee285
Branch: refs/heads/master
Commit: e62ee285265248009c984218ac01aae0473a04c6
Parents: e294761 f973472
Author: Ethan Li <et...@gmail.com>
Authored: Sun Apr 29 22:22:06 2018 -0500
Committer: Ethan Li <et...@gmail.com>
Committed: Sun Apr 29 22:22:06 2018 -0500
----------------------------------------------------------------------
docs/Resource_Aware_Scheduler_overview.md | 4 ++
.../src/jvm/org/apache/storm/Config.java | 15 +++---
.../org/apache/storm/daemon/worker/Worker.java | 9 ++--
.../apache/storm/daemon/worker/WorkerState.java | 19 +++++++
.../java/org/apache/storm/DaemonConfig.java | 1 +
.../org/apache/storm/daemon/nimbus/Nimbus.java | 55 +++++++++-----------
6 files changed, 61 insertions(+), 42 deletions(-)
----------------------------------------------------------------------