You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/05/20 00:06:26 UTC
[1/5] git commit: STORM-295. Add storm configuration to define
JAVA_HOME
Repository: incubator-storm
Updated Branches:
refs/heads/master a09adda06 -> 73dc6245f
STORM-295. Add storm configuration to define JAVA_HOME
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/2d7f9774
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/2d7f9774
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/2d7f9774
Branch: refs/heads/master
Commit: 2d7f9774160a0520087cdf2cfef968debc5eb168
Parents: d2315e9
Author: Suresh Srinivas <su...@yahoo-inc.com>
Authored: Thu Apr 24 14:33:34 2014 -0700
Committer: Suresh Srinivas <su...@yahoo-inc.com>
Committed: Thu Apr 24 16:48:11 2014 -0700
----------------------------------------------------------------------
bin/storm | 37 +++++++++++++++------
conf/storm_env.ini | 30 +++++++++++++++++
storm-dist/binary/src/main/assembly/binary.xml | 5 +++
3 files changed, 61 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2d7f9774/bin/storm
----------------------------------------------------------------------
diff --git a/bin/storm b/bin/storm
index bb92f1c..29df240 100755
--- a/bin/storm
+++ b/bin/storm
@@ -20,9 +20,9 @@ import os
import sys
import random
import subprocess as sub
-import getopt
import re
import shlex
+import ConfigParser
def identity(x):
return x
@@ -34,20 +34,33 @@ def cygpath(x):
lines = output.split("\n")
return lines[0]
-if sys.platform == "cygwin":
- normclasspath = cygpath
-else:
- normclasspath = identity
-
+def init_storm_env():
+ global CLUSTER_CONF_DIR
+ ini_file = os.path.join(CLUSTER_CONF_DIR, 'storm_env.ini')
+ if not os.path.isfile(ini_file):
+ return
+ config = ConfigParser.ConfigParser()
+ config.optionxform = str
+ config.read(ini_file)
+ options = config.options('environment')
+ for option in options:
+ value = config.get('environment', option)
+ os.environ[option] = value
+
+normclasspath = cygpath if sys.platform == 'cygwin' else identity
STORM_DIR = "/".join(os.path.realpath( __file__ ).split("/")[:-2])
USER_CONF_DIR = os.path.expanduser("~/.storm")
CLUSTER_CONF_DIR = STORM_DIR + "/conf"
if (not os.path.isfile(USER_CONF_DIR + "/storm.yaml")):
USER_CONF_DIR = CLUSTER_CONF_DIR
+
+init_storm_env()
+
CONFIG_OPTS = []
CONFFILE = ""
JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', ''))
-
+JAVA_HOME = os.getenv('JAVA_HOME', None)
+JAVA_CMD = 'java' if not JAVA_HOME else os.path.join(JAVA_HOME, 'bin', 'java')
def get_config_opts():
global CONFIG_OPTS
@@ -77,7 +90,8 @@ def get_classpath(extrajars):
def confvalue(name, extrapaths):
global CONFFILE
command = [
- "java", "-client", get_config_opts(), "-Dstorm.conf.file=" + CONFFILE, "-cp", get_classpath(extrapaths), "backtype.storm.command.config_value", name
+ JAVA_CMD, "-client", get_config_opts(), "-Dstorm.conf.file=" + CONFFILE,
+ "-cp", get_classpath(extrapaths), "backtype.storm.command.config_value", name
]
p = sub.Popen(command, stdout=sub.PIPE)
output, errors = p.communicate()
@@ -130,7 +144,7 @@ def parse_args(string):
def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False):
global CONFFILE
all_args = [
- "java", jvmtype, get_config_opts(),
+ JAVA_CMD, jvmtype, get_config_opts(),
"-Dstorm.home=" + STORM_DIR,
"-Djava.library.path=" + confvalue("java.library.path", extrajars),
"-Dstorm.conf.file=" + CONFFILE,
@@ -138,9 +152,10 @@ def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[]
] + jvmopts + [klass] + list(args)
print "Running: " + " ".join(all_args)
if fork:
- os.spawnvp(os.P_WAIT, "java", all_args)
+ os.spawnvp(os.P_WAIT, JAVA_CMD, all_args)
else:
- os.execvp("java", all_args) # replaces the current process and never returns
+ os.execvp(JAVA_CMD, all_args) # replaces the current process and
+ # never returns
def jar(jarfile, klass, *args):
"""Syntax: [storm jar topology-jar-path class ...]
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2d7f9774/conf/storm_env.ini
----------------------------------------------------------------------
diff --git a/conf/storm_env.ini b/conf/storm_env.ini
new file mode 100644
index 0000000..d7af930
--- /dev/null
+++ b/conf/storm_env.ini
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# Environment variables in the following section will be used
+# in storm pytyon script. It override the environment variables
+# set in the shell.
+[environment]
+
+# The java implementation to use. If JAVA_HOME is not found we expect java to be in path
+#JAVA_HOME:home
+
+# JVM options to be used in "storm jar" commad
+#STORM_JAR_JVM_OPTS:
+
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/2d7f9774/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------
diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml
index 6e36704..6f344d1 100644
--- a/storm-dist/binary/src/main/assembly/binary.xml
+++ b/storm-dist/binary/src/main/assembly/binary.xml
@@ -96,6 +96,11 @@
<outputDirectory>/conf</outputDirectory>
<destName>storm.yaml</destName>
</file>
+ <file>
+ <source>${project.basedir}/../../conf/storm_env.ini</source>
+ <outputDirectory>/conf</outputDirectory>
+ <destName>storm_env.ini</destName>
+ </file>
<!-- TODO this should be a generated file from "target" -->
<file>
<source>${project.basedir}/../../VERSION</source>
[3/5] git commit: Merge pull request #1 from ptgoetz/STORM-295
Posted by bo...@apache.org.
Merge pull request #1 from ptgoetz/STORM-295
launch workers using java from JAVA_HOME if set
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/8e0a9765
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/8e0a9765
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/8e0a9765
Branch: refs/heads/master
Commit: 8e0a97652a9eb68b558accf2a205a1cc8a899fbc
Parents: 2d7f977 38b288e
Author: Suresh Srinivas <su...@hortonworks.com>
Authored: Tue Apr 29 10:48:35 2014 -0700
Committer: Suresh Srinivas <su...@hortonworks.com>
Committed: Tue Apr 29 10:48:35 2014 -0700
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 10 +++++++++-
storm-core/test/clj/backtype/storm/supervisor_test.clj | 2 +-
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[5/5] git commit: Added STORM-295 to changelog
Posted by bo...@apache.org.
Added STORM-295 to changelog
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/73dc6245
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/73dc6245
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/73dc6245
Branch: refs/heads/master
Commit: 73dc6245ff0bd53b737df663926d7bd1a93a55c9
Parents: d1159e3
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon May 19 17:02:40 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon May 19 17:02:40 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/73dc6245/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4a8f4ff..e49c908 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.9.2-incubating (unreleased)
+ * STORM-295: Add storm configuration to define JAVA_HOME
* STORM-138: Pluggable serialization for multilang
* STORM-264: Removes references to the deprecated topology.optimize
* STORM-245: implement Stream.localOrShuffle() for trident
[2/5] git commit: launch workers using java from JAVA_HOME if set
Posted by bo...@apache.org.
launch workers using java from JAVA_HOME if set
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/38b288e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/38b288e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/38b288e3
Branch: refs/heads/master
Commit: 38b288e38768c97090cc86083dd1e6915a65658f
Parents: 2d7f977
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Mon Apr 28 16:23:35 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Mon Apr 28 16:23:35 2014 -0400
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 10 +++++++++-
storm-core/test/clj/backtype/storm/supervisor_test.clj | 2 +-
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/38b288e3/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 0ffb888..9ba8156 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -444,6 +444,14 @@
(map sub-fn value)
(-> value sub-fn (.split " ")))))
+(defn java-cmd []
+ (let [java-home (.get (System/getenv) "JAVA_HOME")]
+ (if (nil? java-home)
+ "java"
+ (str java-home file-path-separator "bin" file-path-separator "java")
+ )))
+
+
(defmethod launch-worker
:distributed [supervisor storm-id port worker-id]
(let [conf (:conf supervisor)
@@ -458,7 +466,7 @@
(substitute-worker-childopts s port))
logfilename (str "worker-" port ".log")
command (concat
- ["java" "-server"]
+ [(java-cmd) "-server"]
worker-childopts
topo-worker-childopts
[(str "-Djava.library.path=" (conf JAVA-LIBRARY-PATH))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/38b288e3/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/supervisor_test.clj b/storm-core/test/clj/backtype/storm/supervisor_test.clj
index 54a3ee9..0747b31 100644
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@ -248,7 +248,7 @@
mock-worker-id "fake-worker-id"
mock-cp "mock-classpath"
exp-args-fn (fn [opts topo-opts]
- (concat ["java" "-server"]
+ (concat [(supervisor/java-cmd) "-server"]
opts
topo-opts
["-Djava.library.path="
[4/5] git commit: Merge branch 'STORM-295' of
https://github.com/sureshms/incubator-storm into STORM-295
Posted by bo...@apache.org.
Merge branch 'STORM-295' of https://github.com/sureshms/incubator-storm into STORM-295
Conflicts:
bin/storm
STORM-295: Add storm configuration to define JAVA_HOME
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/d1159e34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/d1159e34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/d1159e34
Branch: refs/heads/master
Commit: d1159e34c1292f7861bd96cdee508d986a50d276
Parents: a09adda 8e0a976
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon May 19 17:01:55 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon May 19 17:01:55 2014 -0500
----------------------------------------------------------------------
bin/storm | 37 ++++++++++++++------
conf/storm_env.ini | 30 ++++++++++++++++
.../clj/backtype/storm/daemon/supervisor.clj | 10 +++++-
.../test/clj/backtype/storm/supervisor_test.clj | 2 +-
storm-dist/binary/src/main/assembly/binary.xml | 5 +++
5 files changed, 71 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/d1159e34/bin/storm
----------------------------------------------------------------------
diff --cc bin/storm
index bb6e40a,29df240..c9885a2
--- a/bin/storm
+++ b/bin/storm
@@@ -20,10 -20,9 +20,10 @@@ import o
import sys
import random
import subprocess as sub
- import getopt
import re
import shlex
+import urllib
+ import ConfigParser
def identity(x):
return x
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/d1159e34/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/d1159e34/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------