You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by zh...@apache.org on 2016/04/01 21:04:43 UTC
[1/2] storm git commit: [STORM-1671] Enable logviewer to delete a dir
without yaml
Repository: storm
Updated Branches:
refs/heads/master 175beec7e -> 68cc70518
[STORM-1671] Enable logviewer to delete a dir without yaml
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/82102837
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/82102837
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/82102837
Branch: refs/heads/master
Commit: 821028374fd09d88d97b3ca3e283da2192f0f2ae
Parents: 3de3afc
Author: zhuol <zh...@yahoo-inc.com>
Authored: Thu Mar 31 13:25:58 2016 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Apr 1 11:07:09 2016 -0500
----------------------------------------------------------------------
.../clj/org/apache/storm/daemon/logviewer.clj | 7 ++++---
.../apache/storm/daemon/DirectoryCleaner.java | 2 +-
.../clj/org/apache/storm/logviewer_test.clj | 20 ++++++++++----------
3 files changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
index 53e8527..2b80fd8 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
@@ -151,9 +151,10 @@
(defn identify-worker-log-dirs [log-dirs]
"return the workerid to worker-log-dir map"
(into {} (for [logdir log-dirs
- :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]
- :when metaFile]
- {(get-worker-id-from-metadata-file metaFile) logdir})))
+ :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]]
+ (if metaFile
+ {(get-worker-id-from-metadata-file metaFile) logdir}
+ {"" logdir})))) ;; an old directory that has no yaml file will be treated as a dead dir for deleting
(defn get-alive-ids
[conf now-secs]
http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
index a51a49e..047e024 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
@@ -138,7 +138,7 @@ public class DirectoryCleaner {
while (!stack.isEmpty() && toDeleteSize > 0) {
File file = stack.pop();
toDeleteSize -= file.length();
- LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getName(), file.length(), file.lastModified());
+ LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getCanonicalPath(), file.length(), file.lastModified());
file.delete();
deletedFiles++;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/82102837/storm-core/test/clj/org/apache/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
index d06c11c..d161f3f 100644
--- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
+++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
@@ -246,24 +246,24 @@
(. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn nil))
(is (= expected (logviewer/identify-worker-log-dirs [port1-dir]))))))))
-
-
(deftest test-get-dead-worker-dirs
- (testing "removes any files of workers that are still alive"
+ (testing "return directories for workers that are not alive"
(let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
- hb (let[lwb (LSWorkerHeartbeat.)]
+ hb (let [lwb (LSWorkerHeartbeat.)]
(.set_time_secs lwb (int 1)) lwb)
id->hb {"42" hb}
now-secs 2
- unexpected-dir (mk-mock-File {:name "dir1" :type :directory})
- expected-dir (mk-mock-File {:name "dir2" :type :directory})
- log-dirs #{unexpected-dir expected-dir}
+ unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory})
+ expected-dir2 (mk-mock-File {:name "dir2" :type :directory})
+ expected-dir3 (mk-mock-File {:name "dir3" :type :directory})
+ log-dirs #{unexpected-dir1 expected-dir2 expected-dir3}
supervisor-util (Mockito/mock SupervisorUtils)]
(with-open [_ (MockedSupervisorUtils. supervisor-util)]
- (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir,
- "007" expected-dir}]
+ (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir1,
+ "007" expected-dir2,
+ "" expected-dir3}] ;; this tests a directory with no yaml file thus no worker id
(. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn id->hb))
- (is (= #{expected-dir}
+ (is (= #{expected-dir2 expected-dir3}
(logviewer/get-dead-worker-dirs conf now-secs log-dirs))))))))
(deftest test-cleanup-fn
[2/2] storm git commit: Merge branch '1671' of
https://github.com/zhuoliu/storm into STORM-1671
Posted by zh...@apache.org.
Merge branch '1671' of https://github.com/zhuoliu/storm into STORM-1671
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/68cc7051
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/68cc7051
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/68cc7051
Branch: refs/heads/master
Commit: 68cc70518f10a9e0ec75fe72aeb2e98624a7b330
Parents: 175beec 8210283
Author: zhuol <zh...@yahoo-inc.com>
Authored: Fri Apr 1 14:04:05 2016 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Fri Apr 1 14:04:05 2016 -0500
----------------------------------------------------------------------
.../clj/org/apache/storm/daemon/logviewer.clj | 7 ++++---
.../apache/storm/daemon/DirectoryCleaner.java | 2 +-
.../clj/org/apache/storm/logviewer_test.clj | 20 ++++++++++----------
3 files changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------