You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by kn...@apache.org on 2016/07/11 21:32:19 UTC
[01/10] storm git commit: Adding user-first classpath.
Repository: storm
Updated Branches:
refs/heads/master f258097d1 -> 05e32892d
Adding user-first classpath.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/55c1afb7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/55c1afb7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/55c1afb7
Branch: refs/heads/master
Commit: 55c1afb777dfdd8795763cf3a943d888a4922386
Parents: 7825a5b
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Tue Jun 21 10:43:47 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Wed Jun 22 10:21:03 2016 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 1 +
.../src/clj/org/apache/storm/daemon/nimbus.clj | 3 +++
storm-core/src/jvm/org/apache/storm/Config.java | 15 +++++++++++++++
.../workermanager/DefaultWorkerManager.java | 12 +++++++++++-
.../src/jvm/org/apache/storm/utils/Utils.java | 16 ++++++++++++++++
5 files changed, 46 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/55c1afb7/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 22e0ba5..5f9b8d2 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -308,3 +308,4 @@ storm.cgroup.hierarchy.name: "storm"
storm.supervisor.cgroup.rootdir: "storm"
storm.cgroup.cgexec.cmd: "/bin/cgexec"
storm.cgroup.memory.limit.tolerance.margin.mb: 128.0
+storm.user.classpath.first.enabled: false
http://git-wip-us.apache.org/repos/asf/storm/blob/55c1afb7/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
index ea1de6e..96370be 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
@@ -1488,6 +1488,9 @@
storm-conf (if (Utils/isZkAuthenticationConfiguredStormServer conf)
storm-conf
(dissoc storm-conf STORM-ZOOKEEPER-TOPOLOGY-AUTH-SCHEME STORM-ZOOKEEPER-TOPOLOGY-AUTH-PAYLOAD))
+ storm-conf (if (conf STORM-USER-CLASSPATH-FIRST-ENABLED)
+ storm-conf
+ (dissoc storm-conf TOPOLOGY-CLASSPATH-FIRST))
total-storm-conf (merge conf storm-conf)
topology (normalize-topology total-storm-conf topology)
storm-cluster-state (:storm-cluster-state nimbus)]
http://git-wip-us.apache.org/repos/asf/storm/blob/55c1afb7/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java
index b725f87..ff32898 100644
--- a/storm-core/src/jvm/org/apache/storm/Config.java
+++ b/storm-core/src/jvm/org/apache/storm/Config.java
@@ -1835,6 +1835,21 @@ public class Config extends HashMap<String, Object> {
public static final String TOPOLOGY_CLASSPATH="topology.classpath";
/**
+ * Topology-specific classpath for the worker child process. This will be *prepended* to
+ * the usual classpath, meaning it can override the Storm classpath. This is for debugging
+ * purposes, and is disabled by default. To allow topologies to be submitted with user-first
+ * classpaths, set the user.classpath.first.enabled config to true.
+ */
+ @isStringOrStringList
+ public static final String TOPOLOGY_CLASSPATH_FIRST="topology.classpath.first";
+
+ /**
+ * Enables user-first classpath. See topology.classpath.first
+ */
+ @isBoolean
+ public static final String STORM_USER_CLASSPATH_FIRST_ENABLED="storm.user.classpath.first.enabled";
+
+ /**
* Topology-specific environment variables for the worker child process.
* This is added to the existing environment (that of the supervisor)
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/55c1afb7/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
index 05ed82b..9fc22ee 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
@@ -347,6 +347,16 @@ public class DefaultWorkerManager implements IWorkerManager {
List<String> topoClasspath = new ArrayList<>();
Object object = stormConf.get(Config.TOPOLOGY_CLASSPATH);
+ // Will be populated only if STORM_USER_CLASSPATH_FIRST_ENABLED is set on Nimbus.
+ // Allowed for extreme debugging.
+ Object topologyClasspathFirst = stormConf.get(Config.TOPOLOGY_CLASSPATH_FIRST);
+ List<String> firstClasspathList = new ArrayList<>();
+ if(topologyClasspathFirst instanceof List) {
+ firstClasspathList.addAll((List<String>)topologyClasspathFirst);
+ } else if (topologyClasspathFirst instanceof String) {
+ firstClasspathList.add((String) topologyClasspathFirst);
+ }
+
if (object instanceof List) {
topoClasspath.addAll((List<String>) object);
} else if (object instanceof String) {
@@ -354,7 +364,7 @@ public class DefaultWorkerManager implements IWorkerManager {
}
LOG.debug("topology specific classpath is {}", object);
- String classPath = Utils.workerClasspath();
+ String classPath = Utils.addToClasspath(firstClasspathList, Arrays.asList(Utils.workerClasspath()));
String classAddPath = Utils.addToClasspath(classPath, Arrays.asList(stormJar));
return Utils.addToClasspath(classAddPath, topoClasspath);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/55c1afb7/storm-core/src/jvm/org/apache/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Utils.java b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
index b1fc51f..eb6755e 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Utils.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
@@ -2092,6 +2092,11 @@ public class Utils {
return _instance.addToClasspathImpl(classpath, paths);
}
+ public static String addToClasspath(Collection<String> classpath,
+ Collection<String> paths) {
+ return _instance.addToClasspathImpl(classpath, paths);
+ }
+
// Non-static impl methods exist for mocking purposes.
public String addToClasspathImpl(String classpath,
Collection<String> paths) {
@@ -2104,6 +2109,17 @@ public class Utils {
return StringUtils.join(l, CLASS_PATH_SEPARATOR);
}
+ public String addToClasspathImpl(Collection<String> classpath,
+ Collection<String> paths) {
+ String basePath = StringUtils.join(classpath, CLASS_PATH_SEPARATOR);
+ if(!basePath.equals("")) {
+ return addToClasspath(basePath, paths);
+ }
+ else {
+ return StringUtils.join(paths, CLASS_PATH_SEPARATOR);
+ }
+ }
+
public static class UptimeComputer {
int startTime = 0;
[02/10] storm git commit: addressing comments.
Posted by kn...@apache.org.
addressing comments.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c7fd8812
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c7fd8812
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c7fd8812
Branch: refs/heads/master
Commit: c7fd8812dcf9501d1da4661660bf951ac3842cff
Parents: 55c1afb
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Wed Jul 6 13:55:36 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Wed Jul 6 13:55:36 2016 -0500
----------------------------------------------------------------------
.../supervisor/workermanager/DefaultWorkerManager.java | 3 ++-
storm-core/src/jvm/org/apache/storm/utils/Utils.java | 13 +++++++------
2 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c7fd8812/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
index 9fc22ee..e8a5990 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
@@ -356,13 +356,14 @@ public class DefaultWorkerManager implements IWorkerManager {
} else if (topologyClasspathFirst instanceof String) {
firstClasspathList.add((String) topologyClasspathFirst);
}
+ LOG.debug("Topology Classpath Prefix: {}", firstClasspathList);
if (object instanceof List) {
topoClasspath.addAll((List<String>) object);
} else if (object instanceof String) {
topoClasspath.add((String) object);
}
- LOG.debug("topology specific classpath is {}", object);
+ LOG.debug("Topology specific classpath is {}", object);
String classPath = Utils.addToClasspath(firstClasspathList, Arrays.asList(Utils.workerClasspath()));
String classAddPath = Utils.addToClasspath(classPath, Arrays.asList(stormJar));
http://git-wip-us.apache.org/repos/asf/storm/blob/c7fd8812/storm-core/src/jvm/org/apache/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Utils.java b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
index eb6755e..3bbad7c 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Utils.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
@@ -2109,15 +2109,16 @@ public class Utils {
return StringUtils.join(l, CLASS_PATH_SEPARATOR);
}
- public String addToClasspathImpl(Collection<String> classpath,
+ public String addToClasspathImpl(Collection<String> classpaths,
Collection<String> paths) {
- String basePath = StringUtils.join(classpath, CLASS_PATH_SEPARATOR);
- if(!basePath.equals("")) {
- return addToClasspath(basePath, paths);
+ List<String> allPaths = new ArrayList<>();
+ if(classpaths != null) {
+ allPaths.addAll(classpaths);
}
- else {
- return StringUtils.join(paths, CLASS_PATH_SEPARATOR);
+ if(paths != null) {
+ allpaths.addAll(classpaths);
}
+ return StringUtils.join(allPaths, CLASS_PATH_SEPARATOR);
}
public static class UptimeComputer {
[03/10] storm git commit: Fixing bug.
Posted by kn...@apache.org.
Fixing bug.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2c924e53
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2c924e53
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2c924e53
Branch: refs/heads/master
Commit: 2c924e532af4583131b4012a5349cadb3bb39626
Parents: c7fd881
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Wed Jul 6 14:38:43 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Wed Jul 6 14:38:43 2016 -0500
----------------------------------------------------------------------
storm-core/src/jvm/org/apache/storm/utils/Utils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2c924e53/storm-core/src/jvm/org/apache/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Utils.java b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
index 3bbad7c..53ca190 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Utils.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
@@ -2116,7 +2116,7 @@ public class Utils {
allPaths.addAll(classpaths);
}
if(paths != null) {
- allpaths.addAll(classpaths);
+ allPaths.addAll(paths);
}
return StringUtils.join(allPaths, CLASS_PATH_SEPARATOR);
}
[08/10] storm git commit: Fixing names.
Posted by kn...@apache.org.
Fixing names.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/01ec4820
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/01ec4820
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/01ec4820
Branch: refs/heads/master
Commit: 01ec4820c3dbb45e9dec0c8aefaf90eeeb6ed9fc
Parents: 9c11598
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Mon Jul 11 16:30:07 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Mon Jul 11 16:30:07 2016 -0500
----------------------------------------------------------------------
storm-core/src/jvm/org/apache/storm/Config.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/01ec4820/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java
index 44f11f2..f27f966 100644
--- a/storm-core/src/jvm/org/apache/storm/Config.java
+++ b/storm-core/src/jvm/org/apache/storm/Config.java
@@ -1838,13 +1838,13 @@ public class Config extends HashMap<String, Object> {
* Topology-specific classpath for the worker child process. This will be *prepended* to
* the usual classpath, meaning it can override the Storm classpath. This is for debugging
* purposes, and is disabled by default. To allow topologies to be submitted with user-first
- * classpaths, set the user.classpath.first.enabled config to true.
+ * classpaths, set the storm.topology.classpath.beginning.enabled config to true.
*/
@isStringOrStringList
public static final String TOPOLOGY_CLASSPATH_BEGINNING="topology.classpath.beginning";
/**
- * Enables user-first classpath. See topology.classpath.first
+ * Enables user-first classpath. See topology.classpath.beginning
*/
@isBoolean
public static final String STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED="storm.topology.classpath.beginning.enabled";
[06/10] storm git commit: Fixing indentation
Posted by kn...@apache.org.
Fixing indentation
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3773d158
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3773d158
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3773d158
Branch: refs/heads/master
Commit: 3773d158c9052757407dcf8a4fa87731923bee21
Parents: 6e55135
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Thu Jul 7 15:01:46 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Thu Jul 7 15:01:46 2016 -0500
----------------------------------------------------------------------
.../daemon/supervisor/workermanager/DefaultWorkerManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3773d158/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
index e8a5990..3e8045d 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
@@ -352,7 +352,7 @@ public class DefaultWorkerManager implements IWorkerManager {
Object topologyClasspathFirst = stormConf.get(Config.TOPOLOGY_CLASSPATH_FIRST);
List<String> firstClasspathList = new ArrayList<>();
if(topologyClasspathFirst instanceof List) {
- firstClasspathList.addAll((List<String>)topologyClasspathFirst);
+ firstClasspathList.addAll((List<String>)topologyClasspathFirst);
} else if (topologyClasspathFirst instanceof String) {
firstClasspathList.add((String) topologyClasspathFirst);
}
[05/10] storm git commit: Adding unit test for new addToClasspath
method
Posted by kn...@apache.org.
Adding unit test for new addToClasspath method
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6e551355
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6e551355
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6e551355
Branch: refs/heads/master
Commit: 6e551355f0a3a5e89aec37b3045a473f3d7f1516
Parents: 8898258
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Wed Jul 6 16:59:49 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Wed Jul 6 16:59:49 2016 -0500
----------------------------------------------------------------------
.../clj/org/apache/storm/supervisor_test.clj | 21 +++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6e551355/storm-core/test/clj/org/apache/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/supervisor_test.clj b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
index 0d6603d..9f63a06 100644
--- a/storm-core/test/clj/org/apache/storm/supervisor_test.clj
+++ b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
@@ -502,7 +502,26 @@
(Matchers/eq full-env)
(Matchers/any)
(Matchers/any)
- (Matchers/any)))))))))
+ (Matchers/any))))))
+
+ (testing "testing addToClasspath(Collection<String>, Collection<String>)"
+ (let [firstpart (java.util.ArrayList. ["a" "b" "c"])
+ secondpart (java.util.ArrayList. ["d" "e" "f"])
+ ^java.util.ArrayList nillist nil]
+ (is (= (Utils/addToClasspath firstpart secondpart)
+ "a:b:c:d:e:f"))
+ (is (= (Utils/addToClasspath [] secondpart)
+ "d:e:f"))
+ (is (= (Utils/addToClasspath nillist secondpart)
+ "d:e:f"))
+ (is (= (Utils/addToClasspath firstpart [])
+ "a:b:c"))
+ (is (= (Utils/addToClasspath firstpart nillist)
+ "a:b:c"))
+ (is (= (Utils/addToClasspath [] [])
+ ""))
+ (is (= (Utils/addToClasspath nillist nillist)
+ "")))))))
(deftest test-worker-launch-command-run-as-user
(testing "*.worker.childopts configuration"
[09/10] storm git commit: Merge branch 'STORM-1916'
Posted by kn...@apache.org.
Merge branch 'STORM-1916'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e2313dc4
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e2313dc4
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e2313dc4
Branch: refs/heads/master
Commit: e2313dc480608588fba7978eba4d73ed9fe45b6a
Parents: f258097 01ec482
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Mon Jul 11 16:31:30 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Mon Jul 11 16:31:30 2016 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 1 +
.../src/clj/org/apache/storm/daemon/nimbus.clj | 3 +++
storm-core/src/jvm/org/apache/storm/Config.java | 15 ++++++++++++++
.../workermanager/DefaultWorkerManager.java | 15 ++++++++++++--
.../src/jvm/org/apache/storm/utils/Utils.java | 17 ++++++++++++++++
.../clj/org/apache/storm/supervisor_test.clj | 21 +++++++++++++++++++-
6 files changed, 69 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e2313dc4/conf/defaults.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e2313dc4/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
----------------------------------------------------------------------
[10/10] storm git commit: Adding STORM-1916 to CHANGELOG
Posted by kn...@apache.org.
Adding STORM-1916 to CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/05e32892
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/05e32892
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/05e32892
Branch: refs/heads/master
Commit: 05e32892dc27c02ea634e5a1a94a55fc0b39e6fc
Parents: e2313dc
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Mon Jul 11 16:32:05 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Mon Jul 11 16:32:05 2016 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/05e32892/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b05a75..0afa9cd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-1916: Add ability for worker-first classpath
* STORM-1954: Large Trident topologies can cause memory issues due to DefaultResourceDeclarer object reading config
* STORM-1913: Additions and Improvements for Trident RAS API
* STORM-1959: Add missing license header to KafkaPartitionOffsetLag
[04/10] storm git commit: addressing comments.
Posted by kn...@apache.org.
addressing comments.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8898258e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8898258e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8898258e
Branch: refs/heads/master
Commit: 8898258e6ea466f96561a0cceb56b1fb81115513
Parents: 2c924e5
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Wed Jul 6 14:40:29 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Wed Jul 6 14:40:29 2016 -0500
----------------------------------------------------------------------
storm-core/src/jvm/org/apache/storm/utils/Utils.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8898258e/storm-core/src/jvm/org/apache/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Utils.java b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
index 53ca190..026ad86 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Utils.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
@@ -2092,9 +2092,9 @@ public class Utils {
return _instance.addToClasspathImpl(classpath, paths);
}
- public static String addToClasspath(Collection<String> classpath,
+ public static String addToClasspath(Collection<String> classpaths,
Collection<String> paths) {
- return _instance.addToClasspathImpl(classpath, paths);
+ return _instance.addToClasspathImpl(classpaths, paths);
}
// Non-static impl methods exist for mocking purposes.
[07/10] storm git commit: Changing config names.
Posted by kn...@apache.org.
Changing config names.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9c115987
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9c115987
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9c115987
Branch: refs/heads/master
Commit: 9c1159871831a2ec4dec4b8170b773c641defab3
Parents: 3773d15
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Fri Jul 8 14:53:41 2016 -0500
Committer: Kyle Nusbaum <Ky...@gmail.com>
Committed: Fri Jul 8 14:53:41 2016 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 2 +-
storm-core/src/clj/org/apache/storm/daemon/nimbus.clj | 4 ++--
storm-core/src/jvm/org/apache/storm/Config.java | 4 ++--
.../daemon/supervisor/workermanager/DefaultWorkerManager.java | 4 ++--
4 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9c115987/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 5f9b8d2..6602999 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -308,4 +308,4 @@ storm.cgroup.hierarchy.name: "storm"
storm.supervisor.cgroup.rootdir: "storm"
storm.cgroup.cgexec.cmd: "/bin/cgexec"
storm.cgroup.memory.limit.tolerance.margin.mb: 128.0
-storm.user.classpath.first.enabled: false
+storm.topology.classpath.beginning.enabled: false
http://git-wip-us.apache.org/repos/asf/storm/blob/9c115987/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
index 96370be..99344c6 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj
@@ -1488,9 +1488,9 @@
storm-conf (if (Utils/isZkAuthenticationConfiguredStormServer conf)
storm-conf
(dissoc storm-conf STORM-ZOOKEEPER-TOPOLOGY-AUTH-SCHEME STORM-ZOOKEEPER-TOPOLOGY-AUTH-PAYLOAD))
- storm-conf (if (conf STORM-USER-CLASSPATH-FIRST-ENABLED)
+ storm-conf (if (conf STORM-TOPOLOGY-CLASSPATH-BEGINNING-ENABLED)
storm-conf
- (dissoc storm-conf TOPOLOGY-CLASSPATH-FIRST))
+ (dissoc storm-conf TOPOLOGY-CLASSPATH-BEGINNING))
total-storm-conf (merge conf storm-conf)
topology (normalize-topology total-storm-conf topology)
storm-cluster-state (:storm-cluster-state nimbus)]
http://git-wip-us.apache.org/repos/asf/storm/blob/9c115987/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java
index ff32898..44f11f2 100644
--- a/storm-core/src/jvm/org/apache/storm/Config.java
+++ b/storm-core/src/jvm/org/apache/storm/Config.java
@@ -1841,13 +1841,13 @@ public class Config extends HashMap<String, Object> {
* classpaths, set the user.classpath.first.enabled config to true.
*/
@isStringOrStringList
- public static final String TOPOLOGY_CLASSPATH_FIRST="topology.classpath.first";
+ public static final String TOPOLOGY_CLASSPATH_BEGINNING="topology.classpath.beginning";
/**
* Enables user-first classpath. See topology.classpath.first
*/
@isBoolean
- public static final String STORM_USER_CLASSPATH_FIRST_ENABLED="storm.user.classpath.first.enabled";
+ public static final String STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED="storm.topology.classpath.beginning.enabled";
/**
* Topology-specific environment variables for the worker child process.
http://git-wip-us.apache.org/repos/asf/storm/blob/9c115987/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
index 3e8045d..ee7fde8 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/workermanager/DefaultWorkerManager.java
@@ -347,9 +347,9 @@ public class DefaultWorkerManager implements IWorkerManager {
List<String> topoClasspath = new ArrayList<>();
Object object = stormConf.get(Config.TOPOLOGY_CLASSPATH);
- // Will be populated only if STORM_USER_CLASSPATH_FIRST_ENABLED is set on Nimbus.
+ // Will be populated only if STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED is set on Nimbus.
// Allowed for extreme debugging.
- Object topologyClasspathFirst = stormConf.get(Config.TOPOLOGY_CLASSPATH_FIRST);
+ Object topologyClasspathFirst = stormConf.get(Config.TOPOLOGY_CLASSPATH_BEGINNING);
List<String> firstClasspathList = new ArrayList<>();
if(topologyClasspathFirst instanceof List) {
firstClasspathList.addAll((List<String>)topologyClasspathFirst);