You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2016/04/04 22:55:57 UTC
[1/4] storm git commit: STORM-1670 Fixed unit tests
Repository: storm
Updated Branches:
refs/heads/1.x-branch 29c19837f -> c0f59bb97
STORM-1670 Fixed unit tests
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e332fc67
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e332fc67
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e332fc67
Branch: refs/heads/1.x-branch
Commit: e332fc67783fc6a7f07021799c42caeb420fec67
Parents: ff33253
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Sat Apr 2 08:00:08 2016 +0530
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Mon Apr 4 16:53:57 2016 -0400
----------------------------------------------------------------------
.../test/clj/org/apache/storm/versioned_store_test.clj | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e332fc67/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/versioned_store_test.clj b/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
index c4d122d..2ee7a27 100644
--- a/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
+++ b/storm-core/test/clj/org/apache/storm/versioned_store_test.clj
@@ -16,6 +16,8 @@
(ns org.apache.storm.versioned-store-test
(:use [clojure test])
(:use [org.apache.storm testing])
+ (:import [java.io File])
+ (:import [org.apache.commons.io FileUtils])
(:import [org.apache.storm.utils VersionedStore]))
(defmacro defvstest [name [vs-sym] & body]
@@ -25,18 +27,25 @@
~@body
))))
+(defn writeToFile [^String file]
+ (FileUtils/writeStringToFile (File. file) (str "time:" (System/currentTimeMillis))))
+
(defvstest test-empty-version [vs]
(let [v (.createVersion vs)]
(.succeedVersion vs v)
+ (writeToFile v)
(is (= 1 (count (.getAllVersions vs))))
(is (= v (.mostRecentVersionPath vs)))
))
(defvstest test-multiple-versions [vs]
- (.succeedVersion vs (.createVersion vs))
+ (let [v (.createVersion vs)]
+ (.succeedVersion vs v)
+ (writeToFile v))
(Thread/sleep 100)
(let [v (.createVersion vs)]
(.succeedVersion vs v)
+ (writeToFile v)
(is (= 2 (count (.getAllVersions vs))))
(is (= v (.mostRecentVersionPath vs)))
[4/4] storm git commit: add STORM-1670 to changelog
Posted by pt...@apache.org.
add STORM-1670 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c0f59bb9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c0f59bb9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c0f59bb9
Branch: refs/heads/1.x-branch
Commit: c0f59bb97309343cc6df485c0911471deb3bc058
Parents: e332fc6
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Mon Apr 4 16:55:47 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Mon Apr 4 16:55:47 2016 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c0f59bb9/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b1f7aad..ea18f03 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 1.0.1
+ * STORM-1670: LocalState#get(String) can throw FileNotFoundException which may result supervisor.clj#sync-processes stop assigning new workers/assignments
* STORM-1677: Test resource files are excluded from source distribution, which makes logviewer-test failing
## 1.0.0
[2/4] storm git commit: STORM-1670 Removed DirectoryStream changes
Posted by pt...@apache.org.
STORM-1670 Removed DirectoryStream changes
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ff33253f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ff33253f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ff33253f
Branch: refs/heads/1.x-branch
Commit: ff33253f1e1ab720b9e8dbd11ce7406ab2e61c2a
Parents: 0bf0559
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Sat Apr 2 00:30:08 2016 +0530
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Mon Apr 4 16:53:57 2016 -0400
----------------------------------------------------------------------
.../org/apache/storm/utils/VersionedStore.java | 53 ++++++++------------
1 file changed, 21 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ff33253f/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java b/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
index 49c736f..b9971df 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
@@ -18,13 +18,9 @@
package org.apache.storm.utils;
import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.io.File;
@@ -125,12 +121,10 @@ public class VersionedStore {
}
HashSet<Long> keepers = new HashSet<Long>(versions);
- try(DirectoryStream<Path> directoryStream = Files.newDirectoryStream(new File(_root).toPath())) {
- for (Path path : directoryStream) {
- Long v = parseVersion(path.toAbsolutePath().toString());
- if (v != null && !keepers.contains(v)) {
- deleteVersion(v);
- }
+ for(String p: listDir(_root)) {
+ Long v = parseVersion(p);
+ if(v!=null && !keepers.contains(v)) {
+ deleteVersion(v);
}
}
}
@@ -139,18 +133,16 @@ public class VersionedStore {
* Sorted from most recent to oldest
*/
public List<Long> getAllVersions() throws IOException {
- List<Long> versions = new ArrayList<Long>();
- try(DirectoryStream<Path> pathDirectoryStream = listDirWithFinishedFiles(_root)) {
- for (Path path : pathDirectoryStream) {
- String absolutePath = path.toAbsolutePath().toString();
- if (absolutePath.endsWith(FINISHED_VERSION_SUFFIX)) {
- versions.add(validateAndGetVersion(absolutePath));
- }
+ List<Long> ret = new ArrayList<Long>();
+ for(String s: listDir(_root)) {
+
+ if(s.endsWith(FINISHED_VERSION_SUFFIX) && new File(s.substring(0, s.length() - FINISHED_VERSION_SUFFIX.length())).exists()) {
+ ret.add(validateAndGetVersion(s));
}
}
- Collections.sort(versions);
- Collections.reverse(versions);
- return versions;
+ Collections.sort(ret);
+ Collections.reverse(ret);
+ return ret;
}
private String tokenPath(long version) {
@@ -182,18 +174,15 @@ public class VersionedStore {
private void mkdirs(String path) throws IOException {
new File(path).mkdirs();
}
-
- /**
- * Return files which have both original and finished versions.
- */
- private DirectoryStream<Path> listDirWithFinishedFiles(String dir) throws IOException {
- return Files.newDirectoryStream(new File(dir).toPath(), new DirectoryStream.Filter<Path>() {
- @Override
- public boolean accept(Path path) throws IOException {
- final String filePath = path.toAbsolutePath().toString();
- return filePath.endsWith(FINISHED_VERSION_SUFFIX) &&
- new File(filePath.substring(0, filePath.length() - FINISHED_VERSION_SUFFIX.length())).exists();
+
+ private List<String> listDir(String dir) throws IOException {
+ List<String> ret = new ArrayList<String>();
+ File[] contents = new File(dir).listFiles();
+ if(contents!=null) {
+ for(File f: contents) {
+ ret.add(f.getAbsolutePath());
}
- });
+ }
+ return ret;
}
}
[3/4] storm git commit: STORM-1670 version store should consider ony
active/finished version files
Posted by pt...@apache.org.
STORM-1670 version store should consider ony active/finished version files
Signed-off-by: P. Taylor Goetz <pt...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0bf05592
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0bf05592
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0bf05592
Branch: refs/heads/1.x-branch
Commit: 0bf05592f4dd8834b99994c0f4c5147925de462f
Parents: 29c1983
Author: Satish Duggana <sd...@hortonworks.com>
Authored: Fri Apr 1 16:29:53 2016 +0530
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Mon Apr 4 16:53:57 2016 -0400
----------------------------------------------------------------------
.../org/apache/storm/utils/VersionedStore.java | 52 ++++++++++++--------
1 file changed, 32 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0bf05592/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java b/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
index bbaf0f3..49c736f 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/VersionedStore.java
@@ -18,9 +18,13 @@
package org.apache.storm.utils;
import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.io.File;
@@ -121,10 +125,12 @@ public class VersionedStore {
}
HashSet<Long> keepers = new HashSet<Long>(versions);
- for(String p: listDir(_root)) {
- Long v = parseVersion(p);
- if(v!=null && !keepers.contains(v)) {
- deleteVersion(v);
+ try(DirectoryStream<Path> directoryStream = Files.newDirectoryStream(new File(_root).toPath())) {
+ for (Path path : directoryStream) {
+ Long v = parseVersion(path.toAbsolutePath().toString());
+ if (v != null && !keepers.contains(v)) {
+ deleteVersion(v);
+ }
}
}
}
@@ -133,15 +139,18 @@ public class VersionedStore {
* Sorted from most recent to oldest
*/
public List<Long> getAllVersions() throws IOException {
- List<Long> ret = new ArrayList<Long>();
- for(String s: listDir(_root)) {
- if(s.endsWith(FINISHED_VERSION_SUFFIX)) {
- ret.add(validateAndGetVersion(s));
+ List<Long> versions = new ArrayList<Long>();
+ try(DirectoryStream<Path> pathDirectoryStream = listDirWithFinishedFiles(_root)) {
+ for (Path path : pathDirectoryStream) {
+ String absolutePath = path.toAbsolutePath().toString();
+ if (absolutePath.endsWith(FINISHED_VERSION_SUFFIX)) {
+ versions.add(validateAndGetVersion(absolutePath));
+ }
}
}
- Collections.sort(ret);
- Collections.reverse(ret);
- return ret;
+ Collections.sort(versions);
+ Collections.reverse(versions);
+ return versions;
}
private String tokenPath(long version) {
@@ -173,15 +182,18 @@ public class VersionedStore {
private void mkdirs(String path) throws IOException {
new File(path).mkdirs();
}
-
- private List<String> listDir(String dir) throws IOException {
- List<String> ret = new ArrayList<String>();
- File[] contents = new File(dir).listFiles();
- if(contents!=null) {
- for(File f: contents) {
- ret.add(f.getAbsolutePath());
+
+ /**
+ * Return files which have both original and finished versions.
+ */
+ private DirectoryStream<Path> listDirWithFinishedFiles(String dir) throws IOException {
+ return Files.newDirectoryStream(new File(dir).toPath(), new DirectoryStream.Filter<Path>() {
+ @Override
+ public boolean accept(Path path) throws IOException {
+ final String filePath = path.toAbsolutePath().toString();
+ return filePath.endsWith(FINISHED_VERSION_SUFFIX) &&
+ new File(filePath.substring(0, filePath.length() - FINISHED_VERSION_SUFFIX.length())).exists();
}
- }
- return ret;
+ });
}
}