You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by da...@apache.org on 2015/08/18 17:50:19 UTC
[01/14] storm git commit: A fix for STORM-976.
Repository: storm
Updated Branches:
refs/heads/master 6f37493e1 -> 3b1b3bfa5
A fix for STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b737c2cc
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b737c2cc
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b737c2cc
Branch: refs/heads/master
Commit: b737c2cceb9611ea10c99450c09fb1a4ad28ab02
Parents: d7155d7
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Thu Aug 13 13:24:26 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Thu Aug 13 13:24:26 2015 -0700
----------------------------------------------------------------------
bin/storm-config.cmd | 4 ++--
bin/storm.py | 22 ++++++++++----------
.../src/clj/backtype/storm/daemon/logviewer.clj | 2 +-
.../clj/backtype/storm/daemon/supervisor.clj | 8 +++----
4 files changed, 18 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b737c2cc/bin/storm-config.cmd
----------------------------------------------------------------------
diff --git a/bin/storm-config.cmd b/bin/storm-config.cmd
index 6794023..c0906e7 100644
--- a/bin/storm-config.cmd
+++ b/bin/storm-config.cmd
@@ -83,10 +83,10 @@ if not defined STORM_LOG_DIR (
)
@rem
-@rem retrieve storm.logback.conf.dir from conf file
+@rem retrieve storm.log4j2.conf.dir from conf file
@rem
-"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" backtype.storm.command.config_value storm.logback.conf.dir > %CMD_TEMP_FILE%
+"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" backtype.storm.command.config_value storm.log4j2.conf.dir > %CMD_TEMP_FILE%
FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
FOR /F "tokens=1,* delims= " %%a in ("%%i") do (
http://git-wip-us.apache.org/repos/asf/storm/blob/b737c2cc/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index a6d08db..597426d 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -76,7 +76,7 @@ if (not os.path.isfile(os.path.join(USER_CONF_DIR, "storm.yaml"))):
STORM_LIB_DIR = os.path.join(STORM_DIR, "lib")
STORM_BIN_DIR = os.path.join(STORM_DIR, "bin")
-STORM_LOG4J_CONF_DIR = os.path.join(STORM_DIR, "log4j2")
+STORM_LOG4J2_CONF_DIR = os.path.join(STORM_DIR, "log4j2")
init_storm_env()
@@ -362,12 +362,12 @@ def repl():
cppaths = [CLUSTER_CONF_DIR]
exec_storm_class("clojure.main", jvmtype="-client", extrajars=cppaths)
-def get_log4j_conf_dir():
+def get_log4j2_conf_dir():
cppaths = [CLUSTER_CONF_DIR]
- storm_log4j_conf_dir = confvalue("storm.logback.conf.dir", cppaths)
- if(storm_log4j_conf_dir == None or storm_log4j_conf_dir == "nil"):
- storm_log4j_conf_dir = STORM_LOG4J_CONF_DIR
- return storm_log4j_conf_dir
+ storm_log4j2_conf_dir = confvalue("storm.log4j2.conf.dir", cppaths)
+ if(storm_log4j2_conf_dir == None or storm_log4j2_conf_dir == "nil"):
+ storm_log4j2_conf_dir = STORM_LOG4J2_CONF_DIR
+ return storm_log4j2_conf_dir
def nimbus(klass="backtype.storm.daemon.nimbus"):
"""Syntax: [storm nimbus]
@@ -381,7 +381,7 @@ def nimbus(klass="backtype.storm.daemon.nimbus"):
cppaths = [CLUSTER_CONF_DIR]
jvmopts = parse_args(confvalue("nimbus.childopts", cppaths)) + [
"-Dlogfile.name=nimbus.log",
- "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml"),
+ "-Dlog4j.configurationFile=" + os.path.join(get_log4j2_conf_dir(), "cluster.xml"),
]
exec_storm_class(
klass,
@@ -402,7 +402,7 @@ def supervisor(klass="backtype.storm.daemon.supervisor"):
cppaths = [CLUSTER_CONF_DIR]
jvmopts = parse_args(confvalue("supervisor.childopts", cppaths)) + [
"-Dlogfile.name=supervisor.log",
- "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml"),
+ "-Dlog4j.configurationFile=" + os.path.join(get_log4j2_conf_dir(), "cluster.xml"),
]
exec_storm_class(
klass,
@@ -424,7 +424,7 @@ def ui():
cppaths = [CLUSTER_CONF_DIR]
jvmopts = parse_args(confvalue("ui.childopts", cppaths)) + [
"-Dlogfile.name=ui.log",
- "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml")
+ "-Dlog4j.configurationFile=" + os.path.join(get_log4j2_conf_dir(), "cluster.xml")
]
exec_storm_class(
"backtype.storm.ui.core",
@@ -446,7 +446,7 @@ def logviewer():
cppaths = [CLUSTER_CONF_DIR]
jvmopts = parse_args(confvalue("logviewer.childopts", cppaths)) + [
"-Dlogfile.name=logviewer.log",
- "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml")
+ "-Dlog4j.configurationFile=" + os.path.join(get_log4j2_conf_dir(), "cluster.xml")
]
exec_storm_class(
"backtype.storm.daemon.logviewer",
@@ -467,7 +467,7 @@ def drpc():
cppaths = [CLUSTER_CONF_DIR]
jvmopts = parse_args(confvalue("drpc.childopts", cppaths)) + [
"-Dlogfile.name=drpc.log",
- "-Dlog4j.configurationFile=" + os.path.join(get_log4j_conf_dir(), "cluster.xml")
+ "-Dlog4j.configurationFile=" + os.path.join(get_log4j2_conf_dir(), "cluster.xml")
]
exec_storm_class(
"backtype.storm.daemon.drpc",
http://git-wip-us.apache.org/repos/asf/storm/blob/b737c2cc/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
index a981d94..6ebbbbd 100644
--- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
@@ -218,7 +218,7 @@ Note that if anything goes wrong, this will throw an Error and exit."
(if (and appender-name appender (instance? RollingFileAppender appender))
(.getParent (File. (.getFileName appender)))
(throw
- (RuntimeException. "Log viewer could not find configured appender, or the appender is not a FileAppender. Please check that the appender name configured in storm and logback agree.")))))
+ (RuntimeException. "Log viewer could not find configured appender, or the appender is not a FileAppender. Please check that the appender name configured in storm and log4j2 agree.")))))
(defnk to-btn-link
"Create a link that is formatted like a button"
http://git-wip-us.apache.org/repos/asf/storm/blob/b737c2cc/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 4b6ac5d..cf0d936 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -661,8 +661,8 @@
storm-conf-file (System/getProperty "storm.conf.file")
storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home file-path-separator "logs"))
storm-conf (read-storm-config)
- storm-log-conf-dir (storm-conf "storm.logback.conf.dir")
- storm-logback-conf-dir (or storm-log-conf-dir (str storm-home file-path-separator "log4j2"))
+ storm-log-conf-dir (storm-conf "storm.log4j2.conf.dir")
+ storm-log4j2-conf-dir (or storm-log-conf-dir (str storm-home file-path-separator "log4j2"))
stormroot (supervisor-stormdist-root conf storm-id)
jlp (jlp stormroot conf)
stormjar (supervisor-stormjar-path stormroot)
@@ -694,7 +694,7 @@
(str "-Dstorm.id=" storm-id)
(str "-Dworker.id=" worker-id)
(str "-Dworker.port=" port)
- (str "-Dlog4j.configurationFile=" storm-logback-conf-dir file-path-separator "worker.xml")
+ (str "-Dlog4j.configurationFile=" storm-log4j2-conf-dir file-path-separator "worker.xml")
"backtype.storm.LogWriter"]
[(java-cmd) "-server"]
worker-childopts
@@ -707,7 +707,7 @@
(str "-Dstorm.options=" storm-options)
(str "-Dstorm.log.dir=" storm-log-dir)
(str "-Dlogging.sensitivity=" logging-sensitivity)
- (str "-Dlog4j.configurationFile=" storm-logback-conf-dir file-path-separator "worker.xml")
+ (str "-Dlog4j.configurationFile=" storm-log4j2-conf-dir file-path-separator "worker.xml")
(str "-Dstorm.id=" storm-id)
(str "-Dworker.id=" worker-id)
(str "-Dworker.port=" port)
[06/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/98f6cd26
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/98f6cd26
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/98f6cd26
Branch: refs/heads/master
Commit: 98f6cd26c5e63af03c411952b2c790564a2e7149
Parents: 962d57b
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Fri Aug 14 16:08:57 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Fri Aug 14 16:08:57 2015 -0700
----------------------------------------------------------------------
.../flux-core/src/test/resources/log4j2.xml | 34 ++++++++++++++++++++
.../flux-core/src/test/resources/logback.xml | 30 -----------------
2 files changed, 34 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/98f6cd26/external/flux/flux-core/src/test/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/external/flux/flux-core/src/test/resources/log4j2.xml b/external/flux/flux-core/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..fb342db
--- /dev/null
+++ b/external/flux/flux-core/src/test/resources/log4j2.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<Configuration scan="true" monitorInterval="30">
+ <Appenders>
+ <Console name="A1" target="SYSTEM_OUT">
+ <PatternLayout>
+ <pattern>%-4r [%t] %-5level %c{1} - %msg%n</pattern>
+ </PatternLayout>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Logger name="org.apache.storm.curator" level="warn"/>
+ <Logger name="org.apache.storm.flux" level="debug"/>
+ <Logger name="org.apache.storm.zookeeper" level="warn"/>
+ <Root level="debug">
+ <appender-ref ref="A1"/>
+ </Root>
+ </Loggers>
+</Configuration>
http://git-wip-us.apache.org/repos/asf/storm/blob/98f6cd26/external/flux/flux-core/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/external/flux/flux-core/src/test/resources/logback.xml b/external/flux/flux-core/src/test/resources/logback.xml
deleted file mode 100644
index 1853b8a..0000000
--- a/external/flux/flux-core/src/test/resources/logback.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration scan="true" scanPeriod="30 seconds">
- <appender name="A1" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%-4r [%t] %-5p %c - %m%n</pattern>
- </encoder>
- </appender>
- <logger name="org.apache.storm.zookeeper" level="WARN"/>
- <logger name="org.apache.storm.curator" level="WARN"/>
- <logger name="org.apache.storm.flux" level="DEBUG"/>
- <root level="DEBUG">
- <appender-ref ref="A1"/>
- </root>
-</configuration>
[11/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/89f48094
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/89f48094
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/89f48094
Branch: refs/heads/master
Commit: 89f480941f0811c4d2f6be2ddafa509386283dd7
Parents: e869d72
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Sun Aug 16 15:15:43 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Sun Aug 16 15:15:43 2015 -0700
----------------------------------------------------------------------
docs/documentation/FAQ.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/89f48094/docs/documentation/FAQ.md
----------------------------------------------------------------------
diff --git a/docs/documentation/FAQ.md b/docs/documentation/FAQ.md
index a183753..a69862e 100644
--- a/docs/documentation/FAQ.md
+++ b/docs/documentation/FAQ.md
@@ -29,8 +29,8 @@ documentation: true
### Halp! I cannot see:
* **my logs** Logs by default go to $STORM_HOME/logs. Check that you have write permissions to that directory. They are configured in
- * log4j2/{cluster, worker}.xml (0.11);
- * logback/cluster.xml (0.9 - 0.10);
+ * log4j2/{cluster, worker}.xml (> 0.9);
+ * logback/cluster.xml (0.9);
* log4j/*.properties in earlier versions (< 0.9).
* **final JVM settings** Add the `-XX+PrintFlagsFinal` commandline option in the childopts (see the conf file)
* **final Java system properties** Add `Properties props = System.getProperties(); props.list(System.out);` near where you build your topology.
[07/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e9db8e2e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e9db8e2e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e9db8e2e
Branch: refs/heads/master
Commit: e9db8e2e60f61548caba4871656c7dbf758bdfaf
Parents: 98f6cd2
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Fri Aug 14 16:11:47 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Fri Aug 14 16:11:47 2015 -0700
----------------------------------------------------------------------
external/flux/flux-core/src/test/resources/log4j2.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e9db8e2e/external/flux/flux-core/src/test/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/external/flux/flux-core/src/test/resources/log4j2.xml b/external/flux/flux-core/src/test/resources/log4j2.xml
index fb342db..f749cb4 100644
--- a/external/flux/flux-core/src/test/resources/log4j2.xml
+++ b/external/flux/flux-core/src/test/resources/log4j2.xml
@@ -16,10 +16,10 @@
limitations under the License.
-->
<Configuration scan="true" monitorInterval="30">
- <Appenders>
+ <Appenders>
<Console name="A1" target="SYSTEM_OUT">
<PatternLayout>
- <pattern>%-4r [%t] %-5level %c{1} - %msg%n</pattern>
+ <pattern>%-4r [%t] %-5level %c{1} - %msg%n</pattern>
</PatternLayout>
</Console>
</Appenders>
[12/14] storm git commit: Merge branch 'master' of
https://github.com/YvonneIronberg/storm into STORM-976-merge
Posted by da...@apache.org.
Merge branch 'master' of https://github.com/YvonneIronberg/storm into STORM-976-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f6e2aa5a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f6e2aa5a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f6e2aa5a
Branch: refs/heads/master
Commit: f6e2aa5a085341d017eefbc43d4c29bf1413e80f
Parents: 6f37493 89f4809
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Tue Aug 18 09:51:57 2015 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Tue Aug 18 09:51:57 2015 -0500
----------------------------------------------------------------------
SECURITY.md | 2 +-
bin/storm-config.cmd | 18 +++++------
bin/storm.py | 22 ++++++-------
conf/defaults.yaml | 1 +
docs/documentation/FAQ.md | 7 ++--
.../flux-core/src/test/resources/log4j2.xml | 34 ++++++++++++++++++++
.../flux-core/src/test/resources/logback.xml | 30 -----------------
.../src/clj/backtype/storm/daemon/logviewer.clj | 2 +-
.../clj/backtype/storm/daemon/supervisor.clj | 9 +++---
storm-core/src/jvm/backtype/storm/Config.java | 6 ++++
10 files changed, 72 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f6e2aa5a/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[09/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/58a4a07e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/58a4a07e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/58a4a07e
Branch: refs/heads/master
Commit: 58a4a07eb3148979f787f31ed90cc9e2e91df564
Parents: abdd8a9
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Fri Aug 14 17:20:37 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Fri Aug 14 17:20:37 2015 -0700
----------------------------------------------------------------------
SECURITY.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/58a4a07e/SECURITY.md
----------------------------------------------------------------------
diff --git a/SECURITY.md b/SECURITY.md
index c406ce2..1278d40 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -441,7 +441,7 @@ The Logviewer daemon now is also responsible for cleaning up old log files for d
| YAML Setting | Description |
|--------------|-------------------------------------|
-| logviewer.cleanup.age.mins | How old (by last modification time) must a worker's log be before that log is considered for clean-up. (Living workers' logs are never cleaned up by the logviewer: Their logs are rolled via log4j2.) |
+| logviewer.cleanup.age.mins | How old (by last modification time) must a worker's log be before that log is considered for clean-up. (Living workers' logs are never cleaned up by the logviewer: Their logs are rolled via standard logging service (e.g. log4j2 in 0.11).) |
| logviewer.cleanup.interval.secs | Interval of time in seconds that the logviewer cleans up worker logs. |
[03/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7ca9b1eb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7ca9b1eb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7ca9b1eb
Branch: refs/heads/master
Commit: 7ca9b1eb04d506770b5f9690301c5c91a5a5a7ae
Parents: 60e6ed7
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Fri Aug 14 11:08:20 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Fri Aug 14 11:08:20 2015 -0700
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/Config.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7ca9b1eb/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 287d8cc..1f46528 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -148,8 +148,8 @@ public class Config extends HashMap<String, Object> {
/**
* A directory that holds configuration files for log4j2.
*/
- public static final String STORM_LOG4J2_CONF_DIR = "storm.log4j2.conf.dir";
- public static final Object STORM_LOG4J2_CONF_DIR_SCHEMA = String.class;
+ public static final String STORM_LOG4J2_CONF_DIR = "storm.log4j2.conf.dir";
+ public static final Object STORM_LOG4J2_CONF_DIR_SCHEMA = String.class;
/**
* A global task scheduler used to assign topologies's tasks to supervisors' wokers.
[04/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e7088c73
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e7088c73
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e7088c73
Branch: refs/heads/master
Commit: e7088c7348e954938ddc3c1187de5e30ba495a5d
Parents: 7ca9b1e
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Fri Aug 14 11:10:05 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Fri Aug 14 11:10:05 2015 -0700
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/Config.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e7088c73/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 1f46528..89e0bfc 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -145,9 +145,9 @@ public class Config extends HashMap<String, Object> {
public static final String STORM_LOCAL_DIR = "storm.local.dir";
public static final Object STORM_LOCAL_DIR_SCHEMA = String.class;
- /**
- * A directory that holds configuration files for log4j2.
- */
+ /**
+ * A directory that holds configuration files for log4j2.
+ */
public static final String STORM_LOG4J2_CONF_DIR = "storm.log4j2.conf.dir";
public static final Object STORM_LOG4J2_CONF_DIR_SCHEMA = String.class;
[08/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/abdd8a98
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/abdd8a98
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/abdd8a98
Branch: refs/heads/master
Commit: abdd8a9855ca0a1e970cee7570f630a9901b63ec
Parents: e9db8e2
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Fri Aug 14 16:13:13 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Fri Aug 14 16:13:13 2015 -0700
----------------------------------------------------------------------
external/flux/flux-core/src/test/resources/log4j2.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/abdd8a98/external/flux/flux-core/src/test/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/external/flux/flux-core/src/test/resources/log4j2.xml b/external/flux/flux-core/src/test/resources/log4j2.xml
index f749cb4..d3333f3 100644
--- a/external/flux/flux-core/src/test/resources/log4j2.xml
+++ b/external/flux/flux-core/src/test/resources/log4j2.xml
@@ -17,9 +17,9 @@
-->
<Configuration scan="true" monitorInterval="30">
<Appenders>
- <Console name="A1" target="SYSTEM_OUT">
+ <Console name="A1" target="SYSTEM_OUT">
<PatternLayout>
- <pattern>%-4r [%t] %-5level %c{1} - %msg%n</pattern>
+ <pattern>%-4r [%t] %-5level %c{1} - %msg%n</pattern>
</PatternLayout>
</Console>
</Appenders>
[10/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e869d722
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e869d722
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e869d722
Branch: refs/heads/master
Commit: e869d722b38b6f949e32f3cfdcaf295a05267731
Parents: 58a4a07
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Sat Aug 15 16:38:40 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Sat Aug 15 16:38:40 2015 -0700
----------------------------------------------------------------------
SECURITY.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e869d722/SECURITY.md
----------------------------------------------------------------------
diff --git a/SECURITY.md b/SECURITY.md
index 1278d40..75836be 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -441,7 +441,7 @@ The Logviewer daemon now is also responsible for cleaning up old log files for d
| YAML Setting | Description |
|--------------|-------------------------------------|
-| logviewer.cleanup.age.mins | How old (by last modification time) must a worker's log be before that log is considered for clean-up. (Living workers' logs are never cleaned up by the logviewer: Their logs are rolled via standard logging service (e.g. log4j2 in 0.11).) |
+| logviewer.cleanup.age.mins | How old (by last modification time) must a worker's log be before that log is considered for clean-up. (Living workers' logs are never cleaned up by the logviewer: Their logs are rolled via some standard logging service (e.g. log4j2 in 0.11).) |
| logviewer.cleanup.interval.secs | Interval of time in seconds that the logviewer cleans up worker logs. |
[14/14] storm git commit: Merge STORM-976
Posted by da...@apache.org.
Merge STORM-976
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3b1b3bfa
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3b1b3bfa
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3b1b3bfa
Branch: refs/heads/master
Commit: 3b1b3bfa595851afd468594079caa7c772dafd48
Parents: 6f37493 1b401e4
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Tue Aug 18 09:56:04 2015 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Tue Aug 18 09:56:04 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
README.markdown | 1 +
SECURITY.md | 2 +-
bin/storm-config.cmd | 18 +++++------
bin/storm.py | 22 ++++++-------
conf/defaults.yaml | 1 +
docs/documentation/FAQ.md | 7 ++--
.../flux-core/src/test/resources/log4j2.xml | 34 ++++++++++++++++++++
.../flux-core/src/test/resources/logback.xml | 30 -----------------
.../src/clj/backtype/storm/daemon/logviewer.clj | 2 +-
.../clj/backtype/storm/daemon/supervisor.clj | 9 +++---
storm-core/src/jvm/backtype/storm/Config.java | 6 ++++
12 files changed, 74 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
[05/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/962d57be
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/962d57be
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/962d57be
Branch: refs/heads/master
Commit: 962d57be14d81c0f9b485182f6dd73bcd84dd50d
Parents: e7088c7
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Fri Aug 14 16:06:28 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Fri Aug 14 16:06:28 2015 -0700
----------------------------------------------------------------------
SECURITY.md | 2 +-
bin/storm-config.cmd | 14 +++++++-------
docs/documentation/FAQ.md | 7 +++++--
3 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/962d57be/SECURITY.md
----------------------------------------------------------------------
diff --git a/SECURITY.md b/SECURITY.md
index c231547..c406ce2 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -441,7 +441,7 @@ The Logviewer daemon now is also responsible for cleaning up old log files for d
| YAML Setting | Description |
|--------------|-------------------------------------|
-| logviewer.cleanup.age.mins | How old (by last modification time) must a worker's log be before that log is considered for clean-up. (Living workers' logs are never cleaned up by the logviewer: Their logs are rolled via logback.) |
+| logviewer.cleanup.age.mins | How old (by last modification time) must a worker's log be before that log is considered for clean-up. (Living workers' logs are never cleaned up by the logviewer: Their logs are rolled via log4j2.) |
| logviewer.cleanup.interval.secs | Interval of time in seconds that the logviewer cleans up worker logs. |
http://git-wip-us.apache.org/repos/asf/storm/blob/962d57be/bin/storm-config.cmd
----------------------------------------------------------------------
diff --git a/bin/storm-config.cmd b/bin/storm-config.cmd
index c0906e7..b839160 100644
--- a/bin/storm-config.cmd
+++ b/bin/storm-config.cmd
@@ -91,26 +91,26 @@ if not defined STORM_LOG_DIR (
FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
FOR /F "tokens=1,* delims= " %%a in ("%%i") do (
if %%a == VALUE: (
- set STORM_LOGBACK_CONFIGURATION_DIR=%%b
+ set STORM_LOG4J2_CONFIGURATION_DIR=%%b
del /F %CMD_TEMP_FILE%)
)
)
)
@rem
-@rem if STORM_LOGBACK_CONFIGURATION_DIR was defined, also set STORM_LOGBACK_CONFIGURATION_FILE
+@rem if STORM_LOG4J2_CONFIGURATION_DIR was defined, also set STORM_LOG4J2_CONFIGURATION_FILE
@rem
-if not %STORM_LOGBACK_CONFIGURATION_DIR% == nil (
- set STORM_LOGBACK_CONFIGURATION_FILE=%STORM_LOGBACK_CONFIGURATION_DIR%\cluster.xml
+if not %STORM_LOG4J2_CONFIGURATION_DIR% == nil (
+ set STORM_LOG4J2_CONFIGURATION_FILE=%STORM_LOG4J2_CONFIGURATION_DIR%\cluster.xml
)
@rem
@rem otherwise, fall back to default
@rem
-if not defined STORM_LOGBACK_CONFIGURATION_FILE (
- set STORM_LOGBACK_CONFIGURATION_FILE=%STORM_HOME%\log4j2\cluster.xml
+if not defined STORM_LOG4J2_CONFIGURATION_FILE (
+ set STORM_LOG4J2_CONFIGURATION_FILE=%STORM_HOME%\log4j2\cluster.xml
)
"%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" backtype.storm.command.config_value java.library.path > %CMD_TEMP_FILE%
@@ -126,7 +126,7 @@ FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
:storm_opts
set STORM_OPTS=-Dstorm.options= -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH%;%JAVA_HOME%\bin;%JAVA_HOME%\lib;%JAVA_HOME%\jre\bin;%JAVA_HOME%\jre\lib
- set STORM_OPTS=%STORM_OPTS% -Dlog4j.configurationFile=%STORM_LOGBACK_CONFIGURATION_FILE%
+ set STORM_OPTS=%STORM_OPTS% -Dlog4j.configurationFile=%STORM_LOG4J2_CONFIGURATION_FILE%
set STORM_OPTS=%STORM_OPTS% -Dstorm.log.dir=%STORM_LOG_DIR%
del /F %CMD_TEMP_FILE%
http://git-wip-us.apache.org/repos/asf/storm/blob/962d57be/docs/documentation/FAQ.md
----------------------------------------------------------------------
diff --git a/docs/documentation/FAQ.md b/docs/documentation/FAQ.md
index b292b2f..a183753 100644
--- a/docs/documentation/FAQ.md
+++ b/docs/documentation/FAQ.md
@@ -28,7 +28,10 @@ documentation: true
### Halp! I cannot see:
-* **my logs** Logs by default go to $STORM_HOME/logs. Check that you have write permissions to that directory. They are configured in the logback/cluster.xml (0.9) and log4j/*.properties in earlier versions.
+* **my logs** Logs by default go to $STORM_HOME/logs. Check that you have write permissions to that directory. They are configured in
+ * log4j2/{cluster, worker}.xml (0.11);
+ * logback/cluster.xml (0.9 - 0.10);
+ * log4j/*.properties in earlier versions (< 0.9).
* **final JVM settings** Add the `-XX+PrintFlagsFinal` commandline option in the childopts (see the conf file)
* **final Java system properties** Add `Properties props = System.getProperties(); props.list(System.out);` near where you build your topology.
@@ -120,4 +123,4 @@ You cannot know that all events are collected -- this is an epistemological chal
* Set a time limit using domain knowledge
* Introduce a _punctuation_: a record known to come after all records in the given time bucket. Trident uses this scheme to know when a batch is complete. If you for instance receive records from a set of sensors, each in order for that sensor, then once all sensors have sent you a 3:02:xx or later timestamp lets you know you can commit.
* When possible, make your process incremental: each value that comes in makes the answer more an more true. A Trident ReducerAggregator is an operator that takes a prior result and a set of new records and returns a new result. This lets the result be cached and serialized to a datastore; if a server drops off line for a day and then comes back with a full day's worth of data in a rush, the old results will be calmly retrieved and updated.
-* Lambda architecture: Record all events into an archival store (S3, HBase, HDFS) on receipt. in the fast layer, once the time window is clear, process the bucket to get an actionable answer, and ignore everything older than the time window. Periodically run a global aggregation to calculate a "correct" answer.
\ No newline at end of file
+* Lambda architecture: Record all events into an archival store (S3, HBase, HDFS) on receipt. in the fast layer, once the time window is clear, process the bucket to get an actionable answer, and ignore everything older than the time window. Periodically run a global aggregation to calculate a "correct" answer.
[13/14] storm git commit: Update CHANGELOG and contributors for
STORM-976
Posted by da...@apache.org.
Update CHANGELOG and contributors for STORM-976
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1b401e4c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1b401e4c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1b401e4c
Branch: refs/heads/master
Commit: 1b401e4cd89d8b04edf3bc9bf4ec5af61a2e533d
Parents: f6e2aa5
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Tue Aug 18 09:55:21 2015 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Tue Aug 18 09:55:21 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
README.markdown | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1b401e4c/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a2ad498..a85c15e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-976: Config storm.logback.conf.dir is specific to previous logging framework
* STORM-995: Fix excessive logging
* STORM-837: HdfsState ignores commits
* STORM-938: storm-hive add a time interval to flush tuples to hive.
http://git-wip-us.apache.org/repos/asf/storm/blob/1b401e4c/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index e965605..84fb9f7 100644
--- a/README.markdown
+++ b/README.markdown
@@ -214,6 +214,7 @@ under the License.
* Randy Gelhausen ([@randerzander](https://github.com/randerzander))
* Gabor Liptak ([@gliptak](https://github.com/glibtak))
* Arun Mahadevan ([@arunmahadevan](https://github.com/arunmahadevan))
+* Yvonne Ironberg ([@YvonneIronberg](https://github.com/YvonneIronberg))
## Acknowledgements
[02/14] storm git commit: STORM-976.
Posted by da...@apache.org.
STORM-976.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/60e6ed73
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/60e6ed73
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/60e6ed73
Branch: refs/heads/master
Commit: 60e6ed73ebf54811f98ea2cd393d784804bf0b96
Parents: b737c2c
Author: YvonneIronberg <Yv...@gmail.com>
Authored: Thu Aug 13 17:09:15 2015 -0700
Committer: YvonneIronberg <Yv...@gmail.com>
Committed: Thu Aug 13 17:09:15 2015 -0700
----------------------------------------------------------------------
conf/defaults.yaml | 1 +
storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 3 +--
storm-core/src/jvm/backtype/storm/Config.java | 6 ++++++
3 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/60e6ed73/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index c44709f..43e71a5 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -23,6 +23,7 @@ java.library.path: "/usr/local/lib:/opt/local/lib:/usr/lib"
### storm.* configs are general configurations
# the local dir is where jars are kept
storm.local.dir: "storm-local"
+storm.log4j2.conf.dir: "log4j2"
storm.zookeeper.servers:
- "localhost"
storm.zookeeper.port: 2181
http://git-wip-us.apache.org/repos/asf/storm/blob/60e6ed73/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index cf0d936..ae2a63f 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -660,8 +660,7 @@
storm-options (System/getProperty "storm.options")
storm-conf-file (System/getProperty "storm.conf.file")
storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home file-path-separator "logs"))
- storm-conf (read-storm-config)
- storm-log-conf-dir (storm-conf "storm.log4j2.conf.dir")
+ storm-log-conf-dir (conf STORM-LOG4J2-CONF-DIR)
storm-log4j2-conf-dir (or storm-log-conf-dir (str storm-home file-path-separator "log4j2"))
stormroot (supervisor-stormdist-root conf storm-id)
jlp (jlp stormroot conf)
http://git-wip-us.apache.org/repos/asf/storm/blob/60e6ed73/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index cc5d967..287d8cc 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -145,6 +145,12 @@ public class Config extends HashMap<String, Object> {
public static final String STORM_LOCAL_DIR = "storm.local.dir";
public static final Object STORM_LOCAL_DIR_SCHEMA = String.class;
+ /**
+ * A directory that holds configuration files for log4j2.
+ */
+ public static final String STORM_LOG4J2_CONF_DIR = "storm.log4j2.conf.dir";
+ public static final Object STORM_LOG4J2_CONF_DIR_SCHEMA = String.class;
+
/**
* A global task scheduler used to assign topologies's tasks to supervisors' wokers.
*