You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/04/22 22:55:42 UTC
[1/6] storm git commit: Deal with STORM-483 exlib issue
Repository: storm
Updated Branches:
refs/heads/master db1f47022 -> 72b80f930
Deal with STORM-483 exlib issue
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/05306d50
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/05306d50
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/05306d50
Branch: refs/heads/master
Commit: 05306d5053ff91bd323c4b54cd246c9f928ca339
Parents: 330e135
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Apr 8 08:57:31 2015 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Apr 8 08:57:31 2015 -0500
----------------------------------------------------------------------
bin/storm.py | 26 ++++++++++++++------
.../clj/backtype/storm/daemon/supervisor.clj | 2 +-
storm-core/src/clj/backtype/storm/util.clj | 18 ++++++++++++++
storm-dist/binary/src/main/assembly/binary.xml | 18 ++++++++++++++
4 files changed, 56 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/05306d50/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index 2cf6c32..17436b6 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -85,6 +85,10 @@ CONFFILE = ""
JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', ''))
JAVA_HOME = os.getenv('JAVA_HOME', None)
JAVA_CMD = 'java' if not JAVA_HOME else os.path.join(JAVA_HOME, 'bin', 'java')
+NIMBUS_EXTRA_CLASSPATHS = os.getenv('NIMBUS_EXTRA_CLASSPATHS', None)
+SUPERVISOR_EXTRA_CLASSPATHS = os.getenv('SUPERVISOR_EXTRA_CLASSPATHS', None)
+STORM_EXT_CLASSPATH = os.getenv('STORM_EXT_CLASSPATH', None)
+STORM_EXT_CLASSPATH_DAEMON = os.getenv('STORM_EXT_CLASSPATH_DAEMON', None)
def get_config_opts():
global CONFIG_OPTS
@@ -105,17 +109,24 @@ def get_jars_full(adir):
ret.append(os.path.join(adir, f))
return ret
-def get_classpath(extrajars):
+def get_classpath(extrajars, daemon=True):
ret = get_jars_full(STORM_DIR)
- ret.extend(get_jars_full(STORM_LIB_DIR))
+ ret.extend(get_jars_full(STORM_DIR + "/lib"))
+ ret.extend(get_jars_full(STORM_DIR + "/extlib"))
+ if daemon:
+ ret.extend(get_jars_full(STORM_DIR + "/extlib-daemon"))
+ if STORM_EXT_CLASSPATH != None:
+ ret.extend(STORM_EXT_CLASSPATH)
+ if daemon and STORM_EXT_CLASSPATH_DAEMON != None:
+ ret.extend(STORM_EXT_CLASSPATH_DAEMON)
ret.extend(extrajars)
return normclasspath(os.pathsep.join(ret))
-def confvalue(name, extrapaths):
+def confvalue(name, extrapaths, daemon=True):
global CONFFILE
command = [
JAVA_CMD, "-client", get_config_opts(), "-Dstorm.conf.file=" + CONFFILE,
- "-cp", get_classpath(extrapaths), "backtype.storm.command.config_value", name
+ "-cp", get_classpath(extrapaths, daemon), "backtype.storm.command.config_value", name
]
p = sub.Popen(command, stdout=sub.PIPE)
output, errors = p.communicate()
@@ -168,7 +179,7 @@ def parse_args(string):
args = [re.compile(r"'((?:[^'\\]|\\.)*)'").sub('\\1', x) for x in args]
return [re.compile(r'\\(.)').sub('\\1', x) for x in args]
-def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False):
+def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False, daemon=True):
global CONFFILE
storm_log_dir = confvalue("storm.log.dir",[CLUSTER_CONF_DIR])
if(storm_log_dir == None or storm_log_dir == "nil"):
@@ -177,9 +188,9 @@ def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[]
"java", jvmtype, get_config_opts(),
"-Dstorm.home=" + STORM_DIR,
"-Dstorm.log.dir=" + storm_log_dir,
- "-Djava.library.path=" + confvalue("java.library.path", extrajars),
+ "-Djava.library.path=" + confvalue("java.library.path", extrajars, daemon),
"-Dstorm.conf.file=" + CONFFILE,
- "-cp", get_classpath(extrajars),
+ "-cp", get_classpath(extrajars, daemon),
] + jvmopts + [klass] + list(args)
print("Running: " + " ".join(all_args))
if fork:
@@ -204,6 +215,7 @@ def jar(jarfile, klass, *args):
jvmtype="-client",
extrajars=[jarfile, USER_CONF_DIR, STORM_BIN_DIR],
args=args,
+ daemon=False,
jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + jarfile])
def kill(*args):
http://git-wip-us.apache.org/repos/asf/storm/blob/05306d50/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index bc8b999..88d6bfc 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -669,7 +669,7 @@
topo-classpath (if-let [cp (storm-conf TOPOLOGY-CLASSPATH)]
[cp]
[])
- classpath (-> (current-classpath)
+ classpath (-> (worker-classpath)
(add-to-classpath [stormjar])
(add-to-classpath topo-classpath))
top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
http://git-wip-us.apache.org/repos/asf/storm/blob/05306d50/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj
index 36dc62c..7c15bae 100644
--- a/storm-core/src/clj/backtype/storm/util.clj
+++ b/storm-core/src/clj/backtype/storm/util.clj
@@ -589,6 +589,24 @@
[]
(System/getProperty "java.class.path"))
+(defn get-full-jars
+ [dir]
+ (map #(str dir file-path-separator %) (filter #(.endsWith % ".jar") (read-dir-contents dir))))
+
+(defn worker-classpath
+ []
+ (let [storm-dir (System/getProperty "storm.home")
+ storm-lib-dir (str storm-dir file-path-separator "lib")
+ storm-conf-dir (if-let [confdir (System/getenv "STORM_CONF_DIR")]
+ confdir
+ (str storm-dir file-path-separator "conf"))
+ storm-extlib-dir (str storm-dir file-path-separator "extlib")
+ extcp (System/getenv "STORM_EXT_CLASSPATH")]
+ (if (nil? storm-dir)
+ (current-classpath)
+ (str/join class-path-separator
+ (concat (get-full-jars storm-lib-dir) (get-full-jars storm-extlib-dir) [extcp] [storm-conf-dir])))))
+
(defn add-to-classpath
[classpath paths]
(if (empty? paths)
http://git-wip-us.apache.org/repos/asf/storm/blob/05306d50/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml
index ff11f40..61a8027 100644
--- a/storm-dist/binary/src/main/assembly/binary.xml
+++ b/storm-dist/binary/src/main/assembly/binary.xml
@@ -136,7 +136,25 @@
<includes>
<include>README.*</include>
</includes>
+ </fileSet>
+
+ <!-- $STORM_HOME/extlib -->
+ <fileSet>
+ <directory></directory>
+ <outputDirectory>extlib</outputDirectory>
+ <includes>
+ <include></include>
+ </includes>
</fileSet>
+ <!-- $STORM_HOME/extlib-daemon, for daemons only -->
+ <fileSet>
+ <directory></directory>
+ <outputDirectory>extlib-daemon</outputDirectory>
+ <includes>
+ <include></include>
+ </includes>
+ </fileSet>
+
</fileSets>
<files>
[4/6] storm git commit: Remove the two unnecessary extra classpaths
Posted by bo...@apache.org.
Remove the two unnecessary extra classpaths
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7301d603
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7301d603
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7301d603
Branch: refs/heads/master
Commit: 7301d60348e1e3e2e6858d984a6981425482f046
Parents: 84b5b08
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Apr 15 09:28:00 2015 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Apr 15 09:28:00 2015 -0500
----------------------------------------------------------------------
bin/storm.py | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7301d603/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index 17436b6..455a4c0 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -85,8 +85,6 @@ CONFFILE = ""
JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', ''))
JAVA_HOME = os.getenv('JAVA_HOME', None)
JAVA_CMD = 'java' if not JAVA_HOME else os.path.join(JAVA_HOME, 'bin', 'java')
-NIMBUS_EXTRA_CLASSPATHS = os.getenv('NIMBUS_EXTRA_CLASSPATHS', None)
-SUPERVISOR_EXTRA_CLASSPATHS = os.getenv('SUPERVISOR_EXTRA_CLASSPATHS', None)
STORM_EXT_CLASSPATH = os.getenv('STORM_EXT_CLASSPATH', None)
STORM_EXT_CLASSPATH_DAEMON = os.getenv('STORM_EXT_CLASSPATH_DAEMON', None)
[2/6] storm git commit: Merge remote-tracking branch 'apache/master'
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c106f133
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c106f133
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c106f133
Branch: refs/heads/master
Commit: c106f1337f84cf7f46be0a133b3bd17b57412d3f
Parents: 05306d5 a6935d4
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Apr 8 09:03:42 2015 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Apr 8 09:03:42 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 11 +
DEVELOPER.md | 4 +-
README.markdown | 1 +
STORM-UI-REST-API.md | 30 +++
.../storm/starter/bolt/AbstractRankerBolt.java | 4 +-
.../storm/starter/bolt/RollingCountAggBolt.java | 1 -
.../storm/starter/bolt/RollingCountBolt.java | 4 +-
.../jvm/storm/starter/util/TupleHelpers.java | 33 ---
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 2 +-
.../storm/hbase/bolt/HBaseLookupBolt.java | 4 +-
.../hbase/trident/state/HBaseMapState.java | 2 +
.../storm/hbase/trident/state/HBaseState.java | 2 +-
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 2 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 2 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 16 +-
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 5 +
external/storm-redis/README.md | 108 ++++++++-
.../redis/topology/PersistentWordCount.java | 35 ---
pom.xml | 2 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 62 ++++-
.../src/clj/backtype/storm/ui/helpers.clj | 15 +-
storm-core/src/jvm/backtype/storm/Config.java | 9 +
.../security/auth/SimpleTransportPlugin.java | 3 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 7 +
.../jvm/backtype/storm/utils/TupleUtils.java | 35 +++
.../src/jvm/backtype/storm/utils/Utils.java | 119 ++++++----
.../trident/topology/TridentBoltExecutor.java | 3 +-
.../storm/security/auth/nimbus_auth_test.clj | 231 ++++++++++---------
28 files changed, 500 insertions(+), 252 deletions(-)
----------------------------------------------------------------------
[5/6] storm git commit: Merge branch 'master' of
https://github.com/zhuoliu/storm into STORM-483
Posted by bo...@apache.org.
Merge branch 'master' of https://github.com/zhuoliu/storm into STORM-483
STORM-483: provide dedicated directories for classpath extension
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/654eae94
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/654eae94
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/654eae94
Branch: refs/heads/master
Commit: 654eae94188e3c7942aa6c3a7dea1581c7cda843
Parents: db1f470 7301d60
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Apr 22 15:46:03 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Apr 22 15:46:03 2015 -0500
----------------------------------------------------------------------
bin/storm.py | 24 ++++++++++++++------
.../clj/backtype/storm/daemon/supervisor.clj | 2 +-
storm-core/src/clj/backtype/storm/util.clj | 18 +++++++++++++++
storm-dist/binary/src/main/assembly/binary.xml | 18 +++++++++++++++
4 files changed, 54 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/654eae94/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
[3/6] storm git commit: Merge remote-tracking branch 'apache/master'
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/84b5b083
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/84b5b083
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/84b5b083
Branch: refs/heads/master
Commit: 84b5b0831d0089bdc85e08cd02f385e07d203c10
Parents: c106f13 2a45a9a
Author: zhuol <zh...@yahoo-inc.com>
Authored: Wed Apr 15 09:26:13 2015 -0500
Committer: zhuol <zh...@yahoo-inc.com>
Committed: Wed Apr 15 09:26:13 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 4 +
STORM-UI-REST-API.md | 44 +-------
storm-core/pom.xml | 5 -
storm-core/src/clj/backtype/storm/converter.clj | 3 +-
.../clj/backtype/storm/daemon/supervisor.clj | 17 ++--
storm-core/src/clj/backtype/storm/testing.clj | 19 ++--
storm-core/src/clj/backtype/storm/ui/core.clj | 13 +--
storm-core/src/clj/backtype/storm/util.clj | 5 -
.../storm/generated/ClusterWorkerHeartbeat.java | 101 ++++++++++++++++++-
storm-core/src/py/__init__.py | 2 +
storm-core/src/py/storm/DistributedRPC.py | 2 +
.../src/py/storm/DistributedRPCInvocations.py | 2 +
storm-core/src/py/storm/Nimbus.py | 2 +
storm-core/src/py/storm/__init__.py | 2 +
storm-core/src/py/storm/constants.py | 2 +
storm-core/src/py/storm/ttypes.py | 19 +++-
storm-core/src/storm.thrift | 1 +
storm-core/src/ui/public/css/style.css | 4 -
storm-core/src/ui/public/js/script.js | 3 +-
.../public/templates/anti-forgery-template.html | 19 ----
storm-core/src/ui/public/topology.html | 8 --
.../storm/security/auth/nimbus_auth_test.clj | 2 +-
22 files changed, 161 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b5b083/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84b5b083/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
[6/6] storm git commit: Add STORM-483 to Changelog
Posted by bo...@apache.org.
Add STORM-483 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/72b80f93
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/72b80f93
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/72b80f93
Branch: refs/heads/master
Commit: 72b80f9302b35bf734a1fb5e219e42bc29274cce
Parents: 654eae9
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Apr 22 15:47:08 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Apr 22 15:47:08 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/72b80f93/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bf8591a..4da34bf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-483: provide dedicated directories for classpath extension
* STORM-764: Have option to compress thrift heartbeat
* STORM-787: test-ns should announce test failures with 'BUILD FAILURE'
* STORM-789: Send more topology context to Multi-Lang components via initial handshake