You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/12/10 18:15:18 UTC
[1/3] storm git commit: [STORM-1370] - Bug fixes for
MultitenantScheduler
Repository: storm
Updated Branches:
refs/heads/master ceb3a0cf7 -> fa6f52588
[STORM-1370] - Bug fixes for MultitenantScheduler
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/82f9d969
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/82f9d969
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/82f9d969
Branch: refs/heads/master
Commit: 82f9d969446898bc6bdbdb03f2b927a55174f97c
Parents: ce2d49b
Author: Boyang Jerry Peng <je...@yahoo-inc.com>
Authored: Fri Dec 4 10:23:10 2015 -0600
Committer: Boyang Jerry Peng <je...@yahoo-inc.com>
Committed: Fri Dec 4 10:23:10 2015 -0600
----------------------------------------------------------------------
.../scheduler/multitenant/IsolatedPool.java | 67 +++++++++++++-------
.../storm/scheduler/multitenant/Node.java | 2 +-
.../scheduler/multitenant_scheduler_test.clj | 52 +++++++++++----
3 files changed, 84 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/82f9d969/storm-core/src/jvm/backtype/storm/scheduler/multitenant/IsolatedPool.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/IsolatedPool.java b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/IsolatedPool.java
index 25a6f25..f079c15 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/IsolatedPool.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/IsolatedPool.java
@@ -92,16 +92,23 @@ public class IsolatedPool extends NodePool {
public void scheduleAsNeeded(NodePool ... lesserPools) {
for (String topId : _topologyIdToNodes.keySet()) {
TopologyDetails td = _tds.get(topId);
- if (_cluster.needsScheduling(td)) {
- LOG.debug("Scheduling topology {}",topId);
- Set<Node> allNodes = _topologyIdToNodes.get(topId);
- Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES);
- int slotsToUse;
- if (nodesRequested == null) {
+ Set<Node> allNodes = _topologyIdToNodes.get(topId);
+ Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES);
+ Integer effectiveNodesRequested = null;
+ if (nodesRequested != null) {
+ effectiveNodesRequested = Math.min(td.getExecutors().size(),
+ +nodesRequested.intValue());
+ }
+ if (_cluster.needsScheduling(td) ||
+ (effectiveNodesRequested != null &&
+ allNodes.size() != effectiveNodesRequested)) {
+ LOG.debug("Scheduling topology {}", topId);
+ int slotsToUse = 0;
+ if (effectiveNodesRequested == null) {
slotsToUse = getNodesForNotIsolatedTop(td, allNodes, lesserPools);
} else {
- slotsToUse = getNodesForIsolatedTop(td, allNodes, lesserPools,
- nodesRequested.intValue());
+ slotsToUse = getNodesForIsolatedTop(td, allNodes, lesserPools,
+ effectiveNodesRequested);
}
//No slots to schedule for some reason, so skip it.
if (slotsToUse <= 0) {
@@ -110,26 +117,16 @@ public class IsolatedPool extends NodePool {
RoundRobinSlotScheduler slotSched =
new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
-
- LinkedList<Node> sortedNodes = new LinkedList<>(allNodes);
- Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
- LOG.debug("Nodes sorted by free space {}", sortedNodes);
+ LOG.debug("Nodes sorted by free space {}", allNodes);
while (true) {
- Node n = sortedNodes.remove();
- if (!slotSched.assignSlotTo(n)) {
+ Node n = findBestNode(allNodes);
+ if (n == null) {
+ LOG.error("No nodes to use to assign topology {}", td.getName());
break;
}
- int freeSlots = n.totalSlotsFree();
- for (int i = 0; i < sortedNodes.size(); i++) {
- if (freeSlots >= sortedNodes.get(i).totalSlotsFree()) {
- sortedNodes.add(i, n);
- n = null;
- break;
- }
- }
- if (n != null) {
- sortedNodes.add(n);
+ if (!slotSched.assignSlotTo(n)) {
+ break;
}
}
}
@@ -138,6 +135,28 @@ public class IsolatedPool extends NodePool {
_cluster.setStatus(topId,"Scheduled Isolated on "+nc+" Nodes");
}
}
+
+ private Node findBestNode(Collection<Node> nodes) {
+ Node ret = null;
+ for(Node node : nodes) {
+ if(ret == null ) {
+ if(node.totalSlotsFree() > 0) {
+ ret = node;
+ }
+ } else {
+ if (node.totalSlotsFree() > 0) {
+ if (node.totalSlotsUsed() < ret.totalSlotsUsed()) {
+ ret = node;
+ } else if (node.totalSlotsUsed() == ret.totalSlotsUsed()) {
+ if(node.totalSlotsFree() > ret.totalSlotsFree()) {
+ ret = node;
+ }
+ }
+ }
+ }
+ }
+ return ret;
+ }
/**
* Get the nodes needed to schedule an isolated topology.
http://git-wip-us.apache.org/repos/asf/storm/blob/82f9d969/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
index 6c2f06b..a8d973c 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
@@ -334,7 +334,7 @@ public class Node {
public static final Comparator<Node> FREE_NODE_COMPARATOR_DEC = new Comparator<Node>() {
@Override
public int compare(Node o1, Node o2) {
- return o2.totalSlotsFree() - o1.totalSlotsFree();
+ return o1.totalSlotsUsed() - o2.totalSlotsUsed();
}
};
}
http://git-wip-us.apache.org/repos/asf/storm/blob/82f9d969/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj b/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
index 4e7e884..b6d06b0 100644
--- a/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
+++ b/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
@@ -132,7 +132,7 @@
executor1 (ed 1)
executor2 (ed 2)
executor3 (ed 3)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"}
(StormTopology.)
2
@@ -169,7 +169,7 @@
executor1 (ed 1)
executor2 (ed 2)
executor3 (ed 3)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"}
(StormTopology.)
5
@@ -208,7 +208,7 @@
executor3 (ed 3)
executor4 (ed 4)
executor5 (ed 5)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"}
(StormTopology.)
5
@@ -247,7 +247,7 @@
executor3 (ed 3)
executor4 (ed 4)
executor5 (ed 5)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"}
(StormTopology.)
5
@@ -296,7 +296,7 @@
executor12 (ed 12)
executor13 (ed 13)
executor14 (ed 14)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"}
(StormTopology.)
2
@@ -375,7 +375,7 @@
executor2 (ed 2)
executor3 (ed 3)
executor4 (ed 4)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"
TOPOLOGY-ISOLATED-MACHINES 4}
(StormTopology.)
@@ -419,7 +419,7 @@
executor2 (ed 2)
executor3 (ed 3)
executor4 (ed 4)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"
TOPOLOGY-ISOLATED-MACHINES 4}
(StormTopology.)
@@ -467,7 +467,7 @@
executor12 (ed 12)
executor13 (ed 13)
executor14 (ed 14)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"}
(StormTopology.)
4
@@ -572,7 +572,7 @@
executor12 (ed 12)
executor13 (ed 13)
executor14 (ed 14)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"}
(StormTopology.)
4
@@ -628,7 +628,7 @@
(deftest test-multitenant-scheduler
(let [supers (gen-supervisors 10)
- topology1 (TopologyDetails. "topology1"
+ topology1 (TopologyDetails. "topology1"
{TOPOLOGY-NAME "topology-name-1"
TOPOLOGY-SUBMITTER-USER "userC"}
(StormTopology.)
@@ -729,8 +729,8 @@
TOPOLOGY-ISOLATED-MACHINES 2
TOPOLOGY-SUBMITTER-USER "userA"}
(StormTopology.)
- 1
- (mk-ed-map [["spout11" 1 2]]))
+ 2
+ (mk-ed-map [["spout11" 1 2]["bolt11" 3 4]]))
topology3 (TopologyDetails. "topology3"
{TOPOLOGY-NAME "topology-name-3"
TOPOLOGY-ISOLATED-MACHINES 1
@@ -829,3 +829,31 @@
(is (= 2 (.size (into #{} (for [slot assigned-slots] (.getNodeId slot))))))
(is (= 2 (.size executors))))
(is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology2"))))))
+
+
+(deftest test-isolated-pool-scheduling-with-nodes-with-different-number-of-slots
+ (let [super1 (SupervisorDetails. "super1" "host2" (list ) (map int (list 1 2 3 4 5)))
+ super2 (SupervisorDetails. "super2" "host2" (list ) (map int (list 1 2 )))
+ supers {"super1" super1 "super2" super2}
+ topology1 (TopologyDetails. "topology1"
+ {TOPOLOGY-NAME "topology-name-1"
+ TOPOLOGY-SUBMITTER-USER "userA"
+ TOPOLOGY-ISOLATED-MACHINES 1}
+ (StormTopology.)
+ 7
+ (mk-ed-map [["spout21" 0 7]]))
+ existing-assignments {"topology1"
+ (SchedulerAssignmentImpl. "topology1"
+ {(ExecutorDetails. 0 0) (WorkerSlot. "super1" 1)
+ (ExecutorDetails. 1 1) (WorkerSlot. "super1" 2)
+ (ExecutorDetails. 2 2) (WorkerSlot. "super1" 3)
+ (ExecutorDetails. 3 3) (WorkerSlot. "super1" 4)
+ (ExecutorDetails. 4 4) (WorkerSlot. "super2" 1)
+ (ExecutorDetails. 5 5) (WorkerSlot. "super2" 2)})}
+ cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments nil)
+ topologies (Topologies. (to-top-map [topology1]))
+ conf {MULTITENANT-SCHEDULER-USER-POOLS {"userA" 2}}
+ scheduler (MultitenantScheduler.)]
+ (.prepare scheduler conf)
+ (.schedule scheduler topologies cluster)
+ (is (= "Scheduled Isolated on 2 Nodes" (.get (.getStatusMap cluster) "topology1")))))
[3/3] storm git commit: Added STORM-1370 to Changelog
Posted by bo...@apache.org.
Added STORM-1370 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fa6f5258
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fa6f5258
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fa6f5258
Branch: refs/heads/master
Commit: fa6f5258825cb7fb69faa3d4d40719bf3cd3e2b6
Parents: ec89262
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Dec 10 11:03:50 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Dec 10 11:03:50 2015 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/fa6f5258/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3e07c59..1164745 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-1370: Bug fixes for MultitenantScheduler
* STORM-1374: fix random failure on WindowManagerTest
* STORM-1040: SQL support for Storm.
* STORM-1364: Log storm version on daemon start
[2/3] storm git commit: Merge branch 'STORM-1370' of
https://github.com/jerrypeng/storm into STORM-1370
Posted by bo...@apache.org.
Merge branch 'STORM-1370' of https://github.com/jerrypeng/storm into STORM-1370
STORM-1370: Bug fixes for MultitenantScheduler
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ec892623
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ec892623
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ec892623
Branch: refs/heads/master
Commit: ec892623a0b266a1bac003a37d15da9cc9700099
Parents: ceb3a0c 82f9d96
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Dec 10 11:03:17 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Dec 10 11:03:17 2015 -0600
----------------------------------------------------------------------
.../scheduler/multitenant/IsolatedPool.java | 67 +++++++++++++-------
.../storm/scheduler/multitenant/Node.java | 2 +-
.../scheduler/multitenant_scheduler_test.clj | 52 +++++++++++----
3 files changed, 84 insertions(+), 37 deletions(-)
----------------------------------------------------------------------