You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/11/13 20:37:17 UTC
[22/50] storm git commit: Scheduler handles already used slot not
reported by supervisor
Scheduler handles already used slot not reported by supervisor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6e3c77a6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6e3c77a6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6e3c77a6
Branch: refs/heads/master
Commit: 6e3c77a69c48c11ce9564b5098953ff7a722e95a
Parents: 7a06ef7
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Thu Aug 28 09:39:36 2014 -0500
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Thu Aug 28 09:39:36 2014 -0500
----------------------------------------------------------------------
.../storm/scheduler/multitenant/Node.java | 20 ++-
.../scheduler/multitenant_scheduler_test.clj | 163 +++++++++++++------
2 files changed, 126 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6e3c77a6/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
index 1c601ca..883c65f 100644
--- a/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
+++ b/storm-core/src/jvm/backtype/storm/scheduler/multitenant/Node.java
@@ -127,10 +127,19 @@ public class Node {
boolean assignInternal(WorkerSlot ws, String topId, boolean dontThrow) {
validateSlot(ws);
if (!_freeSlots.remove(ws)) {
- if (dontThrow) {
- return true;
+ for (Entry<String, Set<WorkerSlot>> topologySetEntry : _topIdToUsedSlots.entrySet()) {
+ if (topologySetEntry.getValue().contains(ws)) {
+ if (dontThrow) {
+ LOG.warn("Worker slot [" + ws + "] can't be assigned to " + topId +
+ ". Its already assigned to " + topologySetEntry.getKey() + ".");
+ return true;
+ }
+ throw new IllegalStateException("Worker slot [" + ws + "] can't be assigned to "
+ + topId + ". Its already assigned to " + topologySetEntry.getKey() + ".");
+ }
}
- throw new IllegalStateException("Assigning a slot that was not free " + ws);
+ LOG.warn("Adding Worker slot [" + ws + "] that was not reported in the supervisor heartbeats," +
+ " but the worker is already running for topology " + topId + ".");
}
Set<WorkerSlot> usedSlots = _topIdToUsedSlots.get(topId);
if (usedSlots == null) {
@@ -291,7 +300,7 @@ public class Node {
//Node ID and supervisor ID are the same.
String id = sup.getId();
boolean isAlive = !cluster.isBlackListed(id);
- LOG.debug("Found a {} Node {} {}",
+ LOG.debug("Found a {} Node {} {}",
new Object[] {isAlive? "living":"dead", id, sup.getAllPorts()});
nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive));
}
@@ -311,7 +320,8 @@ public class Node {
node.addOrphanedSlot(ws);
}
if (node.assignInternal(ws, topId, true)) {
- LOG.warn("Bad scheduling state, "+ws+" assigned multiple workers, unassigning everything...");
+ LOG.warn("Bad scheduling state for topology [" + topId+ "], the slot " +
+ ws + " assigned to multiple workers, un-assigning everything...");
node.free(ws, cluster, true);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6e3c77a6/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj b/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
index b3cdb13..049a0d6 100644
--- a/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
+++ b/storm-core/test/clj/backtype/storm/scheduler/multitenant_scheduler_test.clj
@@ -716,57 +716,116 @@
))
(deftest test-multitenant-scheduler-bad-starting-state
- (let [supers (gen-supervisors 10)
- topology1 (TopologyDetails. "topology1"
- {TOPOLOGY-NAME "topology-name-1"
- TOPOLOGY-SUBMITTER-USER "userC"}
- (StormTopology.)
- 4
- (mk-ed-map [["spout1" 0 5]
- ["bolt1" 5 10]
- ["bolt2" 10 15]
- ["bolt3" 15 20]]))
- topology2 (TopologyDetails. "topology2"
- {TOPOLOGY-NAME "topology-name-2"
- TOPOLOGY-ISOLATED-MACHINES 2
- TOPOLOGY-SUBMITTER-USER "userA"}
- (StormTopology.)
- 4
- (mk-ed-map [["spout11" 0 5]
- ["bolt12" 5 6]
- ["bolt13" 6 7]
- ["bolt14" 7 10]]))
- topology3 (TopologyDetails. "topology3"
- {TOPOLOGY-NAME "topology-name-3"
- TOPOLOGY-ISOLATED-MACHINES 5
- TOPOLOGY-SUBMITTER-USER "userB"}
- (StormTopology.)
- 10
- (mk-ed-map [["spout21" 0 10]
- ["bolt22" 10 20]
- ["bolt23" 20 30]
- ["bolt24" 30 40]]))
- existing-assignments {
- "topology2" (SchedulerAssignmentImpl. "topology2" {(ExecutorDetails. 0 5) (WorkerSlot. "super1" 1)})
- "topology3" (SchedulerAssignmentImpl. "topology3" {(ExecutorDetails. 0 10) (WorkerSlot. "super1" 1)})
- }
- cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
- topologies (Topologies. (to-top-map [topology1 topology2 topology3]))
- conf {MULTITENANT-SCHEDULER-USER-POOLS {"userA" 5 "userB" 5}}
- scheduler (MultitenantScheduler.)]
- (.prepare scheduler conf)
- (.schedule scheduler topologies cluster)
- (let [assignment (.getAssignmentById cluster "topology1")
- assigned-slots (.getSlots assignment)
- executors (.getExecutors assignment)]
- ;; 4 slots on 1 machine, all executors assigned
- (is (= 4 (.size assigned-slots)))
- (is (= 1 (.size (into #{} (for [slot assigned-slots] (.getNodeId slot))))))
- (is (= 20 (.size executors)))
- )
- (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1")))
- (is (= "Scheduled Isolated on 2 Nodes" (.get (.getStatusMap cluster) "topology2")))
- (is (= "Scheduled Isolated on 5 Nodes" (.get (.getStatusMap cluster) "topology3")))
-))
+ (testing "Assiging same worker slot to different topologies is bad state"
+ (let [supers (gen-supervisors 5)
+ topology1 (TopologyDetails. "topology1"
+ {TOPOLOGY-NAME "topology-name-1"
+ TOPOLOGY-SUBMITTER-USER "userC"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout1" 0 1]]))
+ topology2 (TopologyDetails. "topology2"
+ {TOPOLOGY-NAME "topology-name-2"
+ TOPOLOGY-ISOLATED-MACHINES 2
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout11" 1 2]]))
+ topology3 (TopologyDetails. "topology3"
+ {TOPOLOGY-NAME "topology-name-3"
+ TOPOLOGY-ISOLATED-MACHINES 1
+ TOPOLOGY-SUBMITTER-USER "userB"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout21" 2 3]]))
+ worker-slot-with-multiple-assignments (WorkerSlot. "super1" 1)
+ existing-assignments {"topology2" (SchedulerAssignmentImpl. "topology2" {(ExecutorDetails. 1 1) worker-slot-with-multiple-assignments})
+ "topology3" (SchedulerAssignmentImpl. "topology3" {(ExecutorDetails. 2 2) worker-slot-with-multiple-assignments})}
+ cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
+ topologies (Topologies. (to-top-map [topology1 topology2 topology3]))
+ conf {MULTITENANT-SCHEDULER-USER-POOLS {"userA" 2 "userB" 1}}
+ scheduler (MultitenantScheduler.)]
+ (.prepare scheduler conf)
+ (.schedule scheduler topologies cluster)
+ (let [assignment (.getAssignmentById cluster "topology1")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 1 (.size assigned-slots))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1")))
+ (is (= "Scheduled Isolated on 2 Nodes" (.get (.getStatusMap cluster) "topology2")))
+ (is (= "Scheduled Isolated on 1 Nodes" (.get (.getStatusMap cluster) "topology3"))))))
+(deftest test-existing-assignment-slot-not-found-in-supervisor
+ (testing "Scheduler should handle discrepancy when a live supervisor heartbeat does not report slot,
+ but worker heartbeat says its running on that slot"
+ (let [supers (gen-supervisors 1)
+ port-not-reported-by-supervisor 6
+ topology1 (TopologyDetails. "topology1"
+ {TOPOLOGY-NAME "topology-name-1"
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 1
+ (mk-ed-map [["spout11" 0 1]]))
+ existing-assignments {"topology1"
+ (SchedulerAssignmentImpl. "topology1"
+ {(ExecutorDetails. 0 0) (WorkerSlot. "super0" port-not-reported-by-supervisor)})}
+ cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
+ topologies (Topologies. (to-top-map [topology1]))
+ conf {}
+ scheduler (MultitenantScheduler.)]
+ (.prepare scheduler conf)
+ (.schedule scheduler topologies cluster)
+ (let [assignment (.getAssignmentById cluster "topology1")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 1 (.size assigned-slots))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1"))))))
+(deftest test-existing-assignment-slot-on-dead-supervisor
+ (testing "Dead supervisor could have slot with duplicate assignments or slot never reported by supervisor"
+ (let [supers (gen-supervisors 1)
+ dead-supervisor "super1"
+ port-not-reported-by-supervisor 6
+ topology1 (TopologyDetails. "topology1"
+ {TOPOLOGY-NAME "topology-name-1"
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 2
+ (mk-ed-map [["spout11" 0 1]
+ ["bolt12" 1 2]]))
+ topology2 (TopologyDetails. "topology2"
+ {TOPOLOGY-NAME "topology-name-2"
+ TOPOLOGY-SUBMITTER-USER "userA"}
+ (StormTopology.)
+ 2
+ (mk-ed-map [["spout21" 4 5]
+ ["bolt22" 5 6]]))
+ worker-slot-with-multiple-assignments (WorkerSlot. dead-supervisor 1)
+ existing-assignments {"topology1"
+ (SchedulerAssignmentImpl. "topology1"
+ {(ExecutorDetails. 0 0) worker-slot-with-multiple-assignments
+ (ExecutorDetails. 1 1) (WorkerSlot. dead-supervisor 3)})
+ "topology2"
+ (SchedulerAssignmentImpl. "topology2"
+ {(ExecutorDetails. 4 4) worker-slot-with-multiple-assignments
+ (ExecutorDetails. 5 5) (WorkerSlot. dead-supervisor port-not-reported-by-supervisor)})}
+ cluster (Cluster. (nimbus/standalone-nimbus) supers existing-assignments)
+ topologies (Topologies. (to-top-map [topology1 topology2]))
+ conf {}
+ scheduler (MultitenantScheduler.)]
+ (.prepare scheduler conf)
+ (.schedule scheduler topologies cluster)
+ (let [assignment (.getAssignmentById cluster "topology1")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 2 (.size assigned-slots)))
+ (is (= 2 (.size (into #{} (for [slot assigned-slots] (.getNodeId slot))))))
+ (is (= 2 (.size executors))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology1")))
+ (let [assignment (.getAssignmentById cluster "topology2")
+ assigned-slots (.getSlots assignment)
+ executors (.getExecutors assignment)]
+ (is (= 2 (.size assigned-slots)))
+ (is (= 2 (.size (into #{} (for [slot assigned-slots] (.getNodeId slot))))))
+ (is (= 2 (.size executors))))
+ (is (= "Fully Scheduled" (.get (.getStatusMap cluster) "topology2"))))))