You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/05/19 23:32:35 UTC
[1/3] git commit: Removes references to the deprecated
topology.optimize
Repository: incubator-storm
Updated Branches:
refs/heads/master 80f4f7d3f -> 1311a1ddf
Removes references to the deprecated topology.optimize
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/7e175e6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/7e175e6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/7e175e6a
Branch: refs/heads/master
Commit: 7e175e6a81e64048aae356430875ef9085754a97
Parents: d5dee0e
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Mon Mar 24 14:47:25 2014 +0000
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Mon Mar 24 14:47:25 2014 +0000
----------------------------------------------------------------------
conf/defaults.yaml | 1 -
storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 9 ---------
storm-core/src/jvm/backtype/storm/Config.java | 13 -------------
.../test/clj/backtype/storm/integration_test.clj | 9 ++-------
.../test/clj/backtype/storm/multilang_test.clj | 6 +++---
storm-core/test/clj/backtype/storm/nimbus_test.clj | 16 ++++++++--------
6 files changed, 13 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/7e175e6a/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 2dbba24..6bee44a 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -112,7 +112,6 @@ storm.messaging.netty.min_wait_ms: 100
### topology.* configs are for specific executing storms
topology.enable.message.timeouts: true
topology.debug: false
-topology.optimize: true
topology.workers: 1
topology.acker.executors: null
topology.tasks: null
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/7e175e6a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index eaef6c1..bf22a1b 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -300,12 +300,6 @@
[(.getNodeId slot) (.getPort slot)]
)))
-(defn- optimize-topology [topology]
- ;; TODO: create new topology by collapsing bolts into CompoundSpout
- ;; and CompoundBolt
- ;; need to somehow maintain stream/component ids inside tuples
- topology)
-
(defn- setup-storm-code [conf storm-id tmp-jar-location storm-conf topology]
(let [stormroot (master-stormdist-root conf storm-id)]
(FileUtils/forceMkdir (File. stormroot))
@@ -946,9 +940,6 @@
topology)
total-storm-conf (merge conf storm-conf)
topology (normalize-topology total-storm-conf topology)
- topology (if (total-storm-conf TOPOLOGY-OPTIMIZE)
- (optimize-topology topology)
- topology)
storm-cluster-state (:storm-cluster-state nimbus)]
(system-topology! total-storm-conf topology) ;; this validates the structure of the topology
(log-message "Received topology submission for " storm-name " with conf " storm-conf)
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/7e175e6a/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 281ae52..f13d21c 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -500,14 +500,6 @@ public class Config extends HashMap<String, Object> {
public static final String TOPOLOGY_DEBUG = "topology.debug";
public static final Object TOPOLOGY_DEBUG_SCHEMA = Boolean.class;
-
- /**
- * Whether or not the master should optimize topologies by running multiple
- * tasks in a single thread where appropriate.
- */
- public static final String TOPOLOGY_OPTIMIZE = "topology.optimize";
- public static final Object TOPOLOGY_OPTIMIZE_SCHEMA = Boolean.class;
-
/**
* How many processes should be spawned around the cluster to execute this
* topology. Each process will execute some number of tasks as threads within
@@ -834,11 +826,6 @@ public class Config extends HashMap<String, Object> {
setDebug(this, isOn);
}
- @Deprecated
- public void setOptimize(boolean isOn) {
- put(Config.TOPOLOGY_OPTIMIZE, isOn);
- }
-
public static void setNumWorkers(Map conf, int workers) {
conf.put(Config.TOPOLOGY_WORKERS, workers);
}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/7e175e6a/storm-core/test/clj/backtype/storm/integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/integration_test.clj b/storm-core/test/clj/backtype/storm/integration_test.clj
index d54bd40..d96bd90 100644
--- a/storm-core/test/clj/backtype/storm/integration_test.clj
+++ b/storm-core/test/clj/backtype/storm/integration_test.clj
@@ -430,8 +430,7 @@
(deftest test-kryo-decorators-config
(with-simulated-time-local-cluster [cluster
- :daemon-conf {TOPOLOGY-OPTIMIZE false
- TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
+ :daemon-conf {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
TOPOLOGY-KRYO-DECORATORS ["this-is-overriden"]}]
(letlocals
(bind builder (TopologyBuilder.))
@@ -454,8 +453,7 @@
(deftest test-component-specific-config
(with-simulated-time-local-cluster [cluster
- :daemon-conf {TOPOLOGY-OPTIMIZE false
- TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true}]
+ :daemon-conf {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true}]
(letlocals
(bind builder (TopologyBuilder.))
(.setSpout builder "1" (TestPlannerSpout. (Fields. ["conf"])))
@@ -465,7 +463,6 @@
{"fake.config" 123
TOPOLOGY-MAX-TASK-PARALLELISM 20
TOPOLOGY-MAX-SPOUT-PENDING 30
- TOPOLOGY-OPTIMIZE true
TOPOLOGY-KRYO-REGISTER [{"fake.type" "bad.serializer"}
{"fake.type2" "a.serializer"}]
}))
@@ -482,7 +479,6 @@
:mock-sources {"1" [["fake.config"]
[TOPOLOGY-MAX-TASK-PARALLELISM]
[TOPOLOGY-MAX-SPOUT-PENDING]
- [TOPOLOGY-OPTIMIZE]
["fake.config2"]
[TOPOLOGY-KRYO-REGISTER]
]}))
@@ -490,7 +486,6 @@
"fake.config2" 987
TOPOLOGY-MAX-TASK-PARALLELISM 2
TOPOLOGY-MAX-SPOUT-PENDING 30
- TOPOLOGY-OPTIMIZE false
TOPOLOGY-KRYO-REGISTER {"fake.type" "good.serializer"
"fake.type2" "a.serializer"
"fake.type3" "a.serializer3"}}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/7e175e6a/storm-core/test/clj/backtype/storm/multilang_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/multilang_test.clj b/storm-core/test/clj/backtype/storm/multilang_test.clj
index ae288ec..c1cf51d 100644
--- a/storm-core/test/clj/backtype/storm/multilang_test.clj
+++ b/storm-core/test/clj/backtype/storm/multilang_test.clj
@@ -30,7 +30,7 @@
;; )]
;; (submit-local-topology nimbus
;; "test"
-;; {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 20 TOPOLOGY-MESSAGE-TIMEOUT-SECS 3 TOPOLOGY-DEBUG true}
+;; {TOPOLOGY-WORKERS 20 TOPOLOGY-MESSAGE-TIMEOUT-SECS 3 TOPOLOGY-DEBUG true}
;; topology)
;; (Thread/sleep 10000)
;; (.killTopology nimbus "test")
@@ -45,7 +45,7 @@
{"2" (thrift/mk-shell-bolt-spec {"1" :shuffle} "ruby" "tester_bolt.rb" ["word"] :parallelism-hint 1)})]
(submit-local-topology nimbus
"test"
- {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 20 TOPOLOGY-MESSAGE-TIMEOUT-SECS 3 TOPOLOGY-DEBUG true}
+ {TOPOLOGY-WORKERS 20 TOPOLOGY-MESSAGE-TIMEOUT-SECS 3 TOPOLOGY-DEBUG true}
topology)
(Thread/sleep 10000)
(.killTopology nimbus "test")
@@ -61,7 +61,7 @@
)]
(submit-local-topology nimbus
"test"
- {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 20 TOPOLOGY-MESSAGE-TIMEOUT-SECS 3 TOPOLOGY-DEBUG true}
+ {TOPOLOGY-WORKERS 20 TOPOLOGY-MESSAGE-TIMEOUT-SECS 3 TOPOLOGY-DEBUG true}
topology)
(Thread/sleep 10000)
(.killTopology nimbus "test")
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/7e175e6a/storm-core/test/clj/backtype/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/nimbus_test.clj b/storm-core/test/clj/backtype/storm/nimbus_test.clj
index a14767b..1e6554a 100644
--- a/storm-core/test/clj/backtype/storm/nimbus_test.clj
+++ b/storm-core/test/clj/backtype/storm/nimbus_test.clj
@@ -183,7 +183,7 @@
"3" (thrift/mk-bolt-spec {"1" :global} (TestPlannerBolt.) :parallelism-hint 8)
"4" (thrift/mk-bolt-spec {"1" :global "2" :none} (TestPlannerBolt.) :parallelism-hint 4)}
)
- _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 4} topology)
+ _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
task-info (storm-component->task-info cluster "mystorm")]
(check-consistency cluster "mystorm")
;; 3 should be assigned once (if it were optimized, we'd have
@@ -193,7 +193,7 @@
(is (= 4 (count (task-info "2"))))
(is (= 1 (count (task-info "3"))))
(is (= 4 (storm-num-workers state "mystorm")))
- (submit-local-topology nimbus "storm2" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 20} topology2)
+ (submit-local-topology nimbus "storm2" {TOPOLOGY-WORKERS 20} topology2)
(check-consistency cluster "storm2")
(is (= 2 (count (.assignments state nil))))
(let [task-info (storm-component->task-info cluster "storm2")]
@@ -240,13 +240,13 @@
{"2" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 5)
"3" (thrift/mk-bolt-spec {"2" :none} (TestPlannerBolt.))}))
- (submit-local-topology nimbus "noniso" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 4} topology)
+ (submit-local-topology nimbus "noniso" {TOPOLOGY-WORKERS 4} topology)
(advance-cluster-time cluster 1)
(is (= 4 (topology-num-nodes state "noniso")))
(is (= 4 (storm-num-workers state "noniso")))
- (submit-local-topology nimbus "tester1" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 6} topology)
- (submit-local-topology nimbus "tester2" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 6} topology)
+ (submit-local-topology nimbus "tester1" {TOPOLOGY-WORKERS 6} topology)
+ (submit-local-topology nimbus "tester2" {TOPOLOGY-WORKERS 6} topology)
(advance-cluster-time cluster 1)
(bind task-info-tester1 (storm-component->task-info cluster "tester1"))
@@ -284,7 +284,7 @@
{"1" (thrift/mk-spout-spec (TestPlannerSpout. false) :parallelism-hint 3 :conf {TOPOLOGY-TASKS 0})}
{"2" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 1 :conf {TOPOLOGY-TASKS 2})
"3" (thrift/mk-bolt-spec {"2" :none} (TestPlannerBolt.) :conf {TOPOLOGY-TASKS 5})})
- _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 4} topology)
+ _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
task-info (storm-component->task-info cluster "mystorm")]
(check-consistency cluster "mystorm")
(is (= 0 (count (task-info "1"))))
@@ -300,7 +300,7 @@
{"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 3 :conf {TOPOLOGY-TASKS 5})}
{"2" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 8 :conf {TOPOLOGY-TASKS 2})
"3" (thrift/mk-bolt-spec {"2" :none} (TestPlannerBolt.) :parallelism-hint 3)})
- _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 4} topology)
+ _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
task-info (storm-component->task-info cluster "mystorm")
executor-info (->> (storm-component->executor-info cluster "mystorm")
(map-val #(map executor-id->tasks %)))]
@@ -325,7 +325,7 @@
"3" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 2)
"4" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 10)}
)
- _ (submit-local-topology nimbus "test" {TOPOLOGY-OPTIMIZE false TOPOLOGY-WORKERS 7} topology)
+ _ (submit-local-topology nimbus "test" {TOPOLOGY-WORKERS 7} topology)
task-info (storm-component->task-info cluster "test")]
(check-consistency cluster "test")
(is (= 21 (count (task-info "1"))))
[2/3] git commit: Merge branch 'delete-topology-optimize-config' of
https://github.com/d2r/incubator-storm into STORM-254
Posted by bo...@apache.org.
Merge branch 'delete-topology-optimize-config' of https://github.com/d2r/incubator-storm into STORM-254
STORM-264: Removes references to the deprecated topology.optimize
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/fa3802af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/fa3802af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/fa3802af
Branch: refs/heads/master
Commit: fa3802afd1053b2d0494997387bfb8b579bf9489
Parents: 80f4f7d 7e175e6
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon May 19 21:29:44 2014 +0000
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon May 19 21:29:44 2014 +0000
----------------------------------------------------------------------
conf/defaults.yaml | 1 -
storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 9 ---------
storm-core/src/jvm/backtype/storm/Config.java | 13 -------------
.../test/clj/backtype/storm/integration_test.clj | 9 ++-------
.../test/clj/backtype/storm/multilang_test.clj | 6 +++---
storm-core/test/clj/backtype/storm/nimbus_test.clj | 16 ++++++++--------
6 files changed, 13 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/fa3802af/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[3/3] git commit: Added STORM-264 to changelog
Posted by bo...@apache.org.
Added STORM-264 to changelog
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/1311a1dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/1311a1dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/1311a1dd
Branch: refs/heads/master
Commit: 1311a1ddfc097324488f93251062a9ea406f3f77
Parents: fa3802a
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon May 19 21:30:11 2014 +0000
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon May 19 21:30:11 2014 +0000
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/1311a1dd/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 773caf2..a29dfb8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.9.2-incubating (unreleased)
+ * STORM-264: Removes references to the deprecated topology.optimize
* STORM-245: implement Stream.localOrShuffle() for trident
* STORM-317: Add SECURITY.md to release binaries
* STORM-310: Change Twitter authentication