You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by zh...@apache.org on 2016/01/28 18:49:50 UTC
[14/23] storm git commit: Cleanup the comments
Cleanup the comments
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b7374d2b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b7374d2b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b7374d2b
Branch: refs/heads/master
Commit: b7374d2bb4cf49111936ec1cbd2a4f4f22661100
Parents: b4847b7
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Jan 20 09:54:30 2016 -0600
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Jan 20 09:55:25 2016 -0600
----------------------------------------------------------------------
storm-core/src/clj/org/apache/storm/config.clj | 7 +--
.../clj/org/apache/storm/daemon/supervisor.clj | 28 ++----------
.../jvm/org/apache/storm/utils/ConfigUtils.java | 45 +++-----------------
.../clj/org/apache/storm/supervisor_test.clj | 4 --
4 files changed, 11 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b7374d2b/storm-core/src/clj/org/apache/storm/config.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/config.clj b/storm-core/src/clj/org/apache/storm/config.clj
index 32fdee0..3130dcd 100644
--- a/storm-core/src/clj/org/apache/storm/config.clj
+++ b/storm-core/src/clj/org/apache/storm/config.clj
@@ -108,9 +108,7 @@
(defn absolute-storm-local-dir [conf]
(let [storm-home (System/getProperty "storm.home")
- _ (log-message "zliu clojure storm.home is " storm-home)
- path (conf STORM-LOCAL-DIR)
- _ (log-message "zliu clojure path is " path)]
+ path (conf STORM-LOCAL-DIR)]
(if path
(if (is-absolute-path? path) path (str storm-home file-path-separator path))
(str storm-home file-path-separator "storm-local"))))
@@ -189,8 +187,7 @@
(defn supervisor-local-dir
[conf]
- (let [ret (str (absolute-storm-local-dir conf) file-path-separator "supervisor")
- _ (log-message "zliu supervisor-local-dir ret is " ret)]
+ (let [ret (str (absolute-storm-local-dir conf) file-path-separator "supervisor")]
(FileUtils/forceMkdir (File. ret))
ret))
http://git-wip-us.apache.org/repos/asf/storm/blob/b7374d2b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
index 69ee128..b52f316 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -118,11 +118,8 @@
(map-val :master-code-dir assignments-snapshot))
(defn- read-downloaded-storm-ids [conf]
- (let [dir (ConfigUtils/supervisorStormDistRoot conf)
- _ (log-message "zliu java supervisorLocalDir " (ConfigUtils/supervisorLocalDir conf))
- cdir (ConfigUtils/supervisorStormDistRoot conf)
- _ (log-message "zliu jdir is:" dir ",cdir is" cdir ",clj dir equals to nil? " (nil? cdir) "jdir = cdir?" (= dir cdir))]
- (map #(url-decode %) (read-dir-contents dir))) ; (supervisor-stormdist-root conf)));
+ (let [dir (ConfigUtils/supervisorStormDistRoot conf)]
+ (map #(url-decode %) (read-dir-contents dir)))
)
(defn read-worker-heartbeat [conf id]
@@ -140,8 +137,7 @@
(defn read-worker-heartbeats
"Returns map from worker id to heartbeat"
[conf]
- (let [ids (my-worker-ids conf)
- _ (log-message "zliu my-worker-ids are" (prn-str ids))]
+ (let [ids (my-worker-ids conf)]
(into {}
(dofor [id ids]
[id (read-worker-heartbeat conf id)]))
@@ -173,9 +169,7 @@
(let [conf (:conf supervisor)
^LocalState local-state (:local-state supervisor)
id->heartbeat (read-worker-heartbeats conf)
- _ (log-message "zliu id->heartbeat is:" (prn-str id->heartbeat))
- approved-ids (set (keys (ls-approved-workers local-state)))
- _ (log-message "zliu approved-ids is:" (prn-str approved-ids))]
+ approved-ids (set (keys (ls-approved-workers local-state)))]
(into
{}
(dofor [[id hb] id->heartbeat]
@@ -285,7 +279,6 @@
(log-message "Shutting down " (:supervisor-id supervisor) ":" id)
(let [conf (:conf supervisor)
pids (read-dir-contents (ConfigUtils/workerPidsRoot conf id))
- _ (log-message "zliu pids are:" (pr-str pids) ", worker-pids-root is:" (ConfigUtils/workerPidsRoot conf id))
thread-pid (@(:worker-thread-pids-atom supervisor) id)
shutdown-sleep-secs (conf SUPERVISOR-WORKER-SHUTDOWN-SLEEP-SECS)
as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
@@ -383,7 +376,6 @@
(log-message "Launching worker with assignment "
(get-worker-assignment-helper-msg assignment supervisor port id))
(local-mkdirs (ConfigUtils/workerPidsRoot conf id))
- (log-message "zliu create worker heartbeatroot as " (ConfigUtils/workerHeartbeatsRoot conf id))
(local-mkdirs (ConfigUtils/workerHeartbeatsRoot conf id))
(launch-worker supervisor
(:storm-id assignment)
@@ -401,9 +393,6 @@
^LocalState local-state (:local-state supervisor)
storm-cluster-state (:storm-cluster-state supervisor)
assigned-executors (defaulted (ls-local-assignments local-state) {})
- _ (log-message "zliu storm-cluster-state is " (prn-str storm-cluster-state))
- _ (log-message "zliu local-state is " (prn-str local-state))
- _ (log-message "zliu assigned-executors is " (prn-str assigned-executors))
now (current-time-secs)
allocated (read-allocated-workers supervisor assigned-executors now)
keepers (filter-val
@@ -428,9 +417,6 @@
(log-debug "Syncing processes")
(log-debug "Assigned executors: " assigned-executors)
(log-debug "Allocated: " allocated)
- (log-message "zliu Syncing processes")
- (log-message "zliu allocated is " (prn-str allocated))
- (log-message "zliu new-worker-ids is " (prn-str new-worker-ids))
(doseq [[id [state heartbeat]] allocated]
(when (not= :valid state)
(log-message
@@ -1063,21 +1049,15 @@
(str storm-home file-path-separator storm-log-conf-dir))
(str storm-home file-path-separator "log4j2"))
stormroot (ConfigUtils/supervisorStormDistRoot conf storm-id)
- _ (log-message "zliu stormroot: " stormroot)
jlp (jlp stormroot conf)
stormjar (ConfigUtils/supervisorStormJarPath stormroot)
- _ (log-message "zliu stormjar: " stormjar)
storm-conf (clojurify-structure (ConfigUtils/readSupervisorStormConf conf storm-id))
topo-classpath (if-let [cp (storm-conf TOPOLOGY-CLASSPATH)]
[cp]
[])
- _ (log-message "zliu worker-classpath: " (worker-classpath))
- _ (log-message "zliu stormjar: " stormjar)
- _ (log-message "zliu topo-classpath: " topo-classpath)
classpath (-> (worker-classpath)
(add-to-classpath [stormjar])
(add-to-classpath topo-classpath))
- _ (log-message "zliu classpath" classpath)
top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
mem-onheap (if (and mem-onheap (> mem-onheap 0)) ;; not nil and not zero
(int (Math/ceil mem-onheap)) ;; round up
http://git-wip-us.apache.org/repos/asf/storm/blob/b7374d2b/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java b/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
index dcff7c6..bb762e4 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
@@ -42,7 +42,6 @@ public class ConfigUtils {
public static final String FILE_SEPARATOR = File.separator;
public final static String LOG_DIR;
- // TODO: what is the different between ConfigUtils/LOG_DIR and config/LOG-DIR, the first fails tests
static {
String dir;
Map conf;
@@ -64,28 +63,6 @@ public class ConfigUtils {
}
}
- //public final static String WORKER_DATA_SUBDIR = "worker_shared_data";
-
- /*
- public static String getLogDir() {
- String dir;
- Map conf;
- if (System.getProperty("storm.log.dir") != null) {
- dir = System.getProperty("storm.log.dir");
- } else if ((conf = readStormConfig()).get("storm.log.dir") != null) {
- dir = String.valueOf(conf.get("storm.log.dir"));
- } else {
- dir = System.getProperty("storm.home") + FILE_SEPARATOR + "logs";
- }
- try {
- return new File(dir).getCanonicalPath();
- } catch (IOException ex) {
- throw new IllegalArgumentException("Illegal storm.log.dir in conf: " + dir);
- }
- }
- */
-
-
public static String clojureConfigName(String name) {
return name.toUpperCase().replace("_", "-");
}
@@ -136,11 +113,9 @@ public class ConfigUtils {
throw new IllegalArgumentException("Illegal topology.stats.sample.rate in conf: " + rate);
}
- // public static mkStatsSampler // depends on Utils.evenSampler() TODO, this is sth we have to do
-
+ // public static mkStatsSampler // depends on Utils.evenSampler() TODO, this is sth we need to do after util
// public static readDefaultConfig // depends on Utils.clojurifyStructure and Utils.readDefaultConfig // TODO
-
- // validate-configs-with-schemas is just a wrapper of ConfigValidation.validateFields(conf) TODO
+ // validate-configs-with-schemas is just a wrapper of ConfigValidation.validateFields(conf)
//For testing only
// for java
@@ -166,7 +141,7 @@ public class ConfigUtils {
if (mockedStormConfig != null) return mockedStormConfig;
Map conf = Utils.readStormConfig();
ConfigValidation.validateFields(conf);
- return conf; // TODO, should this be clojurify-sturecture and then return? Otherwise, the clj files who call it fail
+ return conf;
}
public static Map readYamlConfig(String name, boolean mustExist) {
@@ -180,14 +155,11 @@ public class ConfigUtils {
}
public static String absoluteStormLocalDir(Map conf) {
- LOG.info("zliu conf map is " + conf);
String stormHome = System.getProperty("storm.home");
- LOG.info("zliu stormhome is " + stormHome);
String localDir = (String) conf.get(Config.STORM_LOCAL_DIR);
if (localDir == null) {
return (stormHome + FILE_SEPARATOR + "storm-local");
} else {
- LOG.info("zliu java local dir is " + localDir + ", isAbsolute:" + (new File(localDir).isAbsolute()));
if (new File(localDir).isAbsolute()) {
return localDir;
} else {
@@ -267,7 +239,7 @@ public class ConfigUtils {
public static Map readSupervisorStormConfGivenPath(Map conf, String stormConfPath) throws IOException {
Map ret = new HashMap(conf);
- ret.putAll(Utils.fromCompressedJsonConf(FileUtils.readFileToByteArray(new File(stormConfPath)))); // TODO we do not need clojurify-structure, right?
+ ret.putAll(Utils.fromCompressedJsonConf(FileUtils.readFileToByteArray(new File(stormConfPath))));
return ret;
}
@@ -357,7 +329,6 @@ public class ConfigUtils {
}
private static Map mockedSupervisorStormDistRoot = null;
public static String supervisorStormDistRoot(Map conf) throws IOException {
- LOG.info("zliu supervisorStormDistRoot resl is: " + stormDistPath(supervisorLocalDir(conf)) + "mocked set is " + mockedSupervisorStormDistRoot);
if (mockedSupervisorStormDistRoot != null) {
return null;
}
@@ -519,8 +490,6 @@ public class ConfigUtils {
public static String getWorkerUser(Map conf, String workerId) {
LOG.info("GET worker-user for {}", workerId);
File file = new File(workerUserFile(conf, workerId));
- //LOG.info("zliu to read access worker user file: " + (file.getCanonicalPath()));
- LOG.info("zliu this file is existed?" + file.exists());
try (InputStream in = new FileInputStream(file);
Reader reader = new InputStreamReader(in);
@@ -578,7 +547,6 @@ public class ConfigUtils {
}
LOG.info("SET worker-user {} {}", workerId, user);
File file = new File(workerUserFile(conf, workerId));
- LOG.info("zliu SET worker-user to create worker file:" + file.getCanonicalPath());
file.getParentFile().mkdirs();
try (FileWriter fw = new FileWriter(file);
@@ -663,9 +631,6 @@ public class ConfigUtils {
}
public static String workerRoot(Map conf) {
- LOG.info("zliu workers root's current listFiles are:");
- File r = new File((absoluteStormLocalDir(conf) + FILE_SEPARATOR + "workers")); //TODO delete me
- if (r.exists()) for (File f : r.listFiles()) {LOG.info(f.getName());} //TODO delete me
return (absoluteStormLocalDir(conf) + FILE_SEPARATOR + "workers");
}
@@ -743,4 +708,4 @@ public class ConfigUtils {
Collections.sort(ret);
return ret;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/b7374d2b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/supervisor_test.clj b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
index e5a3bdf..e0dbb1a 100644
--- a/storm-core/test/clj/org/apache/storm/supervisor_test.clj
+++ b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
@@ -315,7 +315,6 @@
exp-args (exp-args-fn ["-Dfoo=bar" "-Xmx1024m"]
["-Dkau=aux" "-Xmx2048m"]
mock-cp)
- _ (log-message "zliu testing 1, exp-args is: " exp-args)
mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
WORKER-CHILDOPTS string-opts}}
mocked-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
@@ -341,7 +340,6 @@
(let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
exp-args (exp-args-fn list-opts topo-list-opts mock-cp)
- _ (log-message "zliu testing 2, exp-args is: " exp-args)
mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
WORKER-CHILDOPTS list-opts}}
mocked-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
@@ -366,7 +364,6 @@
(testing "testing topology.classpath is added to classpath"
(let [topo-cp (str file-path-separator "any" file-path-separator "path")
exp-args (exp-args-fn [] [] (add-to-classpath mock-cp [topo-cp]))
- _ (log-message "zliu testing 3, exp-args is: " exp-args ", str is " (pr-str exp-args))
mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}
mocked-supervisor-storm-conf {TOPOLOGY-CLASSPATH topo-cp}]
(with-open [mock1 (org.apache.storm.utils.ConfigUtils$SetMockedSupervisorStormDistRoot. {})
@@ -390,7 +387,6 @@
(let [topo-env {"THISVAR" "somevalue" "THATVAR" "someothervalue"}
full-env (merge topo-env {"LD_LIBRARY_PATH" nil})
exp-args (exp-args-fn [] [] mock-cp)
- _ (log-message "zliu testing 4, exp-args is: " exp-args)
mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}
mocked-supervisor-storm-conf {TOPOLOGY-ENVIRONMENT topo-env}]
(with-open [mock1 (org.apache.storm.utils.ConfigUtils$SetMockedSupervisorStormDistRoot. {})