You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2016/02/23 21:53:48 UTC
[1/3] storm git commit: [STORM-1552] Fix topology event sampling log
dir
Repository: storm
Updated Branches:
refs/heads/1.x-branch 4a1b879f6 -> 63bac16c2
[STORM-1552] Fix topology event sampling log dir
Currently the events are logged under "storm-local/workers-artifacts/{storm-id}/{port}/events.log".
and the "events" link in UI does not display the log file.
The events.log should be kept under "logs/workers-artifacts/{storm-id}/{port}/events.log"
so that its viewable via logviewer.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8d609615
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8d609615
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8d609615
Branch: refs/heads/1.x-branch
Commit: 8d6096154ac30bbd3bc390604b6b9376a740c83e
Parents: 4a1b879
Author: Arun Mahadevan <ai...@hortonworks.com>
Authored: Mon Feb 15 20:41:03 2016 +0530
Committer: Arun Mahadevan <ai...@hortonworks.com>
Committed: Mon Feb 22 00:04:51 2016 +0530
----------------------------------------------------------------------
.../storm/metric/FileBasedEventLogger.java | 31 +++++++++++---------
1 file changed, 17 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8d609615/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java b/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
index a56a596..076a68c 100644
--- a/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
+++ b/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
@@ -17,6 +17,7 @@
*/
package org.apache.storm.metric;
+import org.apache.storm.Config;
import org.apache.storm.task.TopologyContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +46,7 @@ public class FileBasedEventLogger implements IEventLogger {
private void initLogWriter(Path logFilePath) {
try {
- LOG.info("logFilePath {}", logFilePath);
+ LOG.info("Event log path {}", logFilePath);
eventLogPath = logFilePath;
eventLogWriter = Files.newBufferedWriter(eventLogPath, StandardCharsets.UTF_8, StandardOpenOption.CREATE,
StandardOpenOption.WRITE, StandardOpenOption.APPEND);
@@ -76,33 +77,35 @@ public class FileBasedEventLogger implements IEventLogger {
scheduler.scheduleAtFixedRate(task, FLUSH_INTERVAL_MILLIS, FLUSH_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
}
+ private String getLogDir(Map stormConf) {
+ String logDir;
+ if ((logDir = System.getProperty("storm.log.dir")) == null
+ && (logDir = (String) stormConf.get("storm.log.dir")) == null) {
+ logDir = Paths.get(System.getProperty("storm.home"), "logs").toString();
+ }
+ return logDir;
+ }
@Override
public void prepare(Map stormConf, TopologyContext context) {
- String logDir; // storm local directory
+ String workersArtifactDir; // workers artifact directory
String stormId = context.getStormId();
int port = context.getThisWorkerPort();
- if ((logDir = System.getProperty("storm.local.dir")) == null &&
- (logDir = (String)stormConf.get("storm.local.dir")) == null) {
- String msg = "Could not determine the directory to log events.";
- LOG.error(msg);
- throw new RuntimeException(msg);
- } else {
- LOG.info("FileBasedEventLogger log directory {}.", logDir);
+ if ((workersArtifactDir = (String) stormConf.get(Config.STORM_WORKERS_ARTIFACTS_DIR)) == null) {
+ workersArtifactDir = "workers-artifacts";
}
-
/*
* Include the topology name & worker port in the file name so that
* multiple event loggers can log independently.
*/
- Path path = Paths.get(logDir, "workers-artifacts", stormId, Integer.toString(port), "events.log");
+ Path path = Paths.get(workersArtifactDir, stormId, Integer.toString(port), "events.log");
if (!path.isAbsolute()) {
- path = Paths.get(System.getProperty("storm.home"), logDir, "workers-artifacts",
- stormId, Integer.toString(port), "events.log");
+ path = Paths.get(getLogDir(stormConf), workersArtifactDir,
+ stormId, Integer.toString(port), "events.log");
}
File dir = path.toFile().getParentFile();
if (!dir.exists()) {
- dir.mkdirs();
+ dir.mkdirs();
}
initLogWriter(path);
setUpFlushTask();
[3/3] storm git commit: Added STORM-1552 to CHANGELOG.
Posted by sr...@apache.org.
Added STORM-1552 to CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/63bac16c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/63bac16c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/63bac16c
Branch: refs/heads/1.x-branch
Commit: 63bac16c2a27c3a3d0c372dc0925f4f606ffe9dd
Parents: f3baed0
Author: Sriharsha Chintalapani <ha...@hortonworks.com>
Authored: Tue Feb 23 12:52:23 2016 -0800
Committer: Sriharsha Chintalapani <ha...@hortonworks.com>
Committed: Tue Feb 23 12:52:23 2016 -0800
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/63bac16c/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d85c002..319dc85 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 1.0.0
+ * STORM-1552: Fix topology event sampling log dir
* STORM-1511: min/max operations support on a trident stream
* STORM-1522: REST API throws invalid worker log links
* STORM-1532: Fix readCommandLineOpts to parse JSON correctly
[2/3] storm git commit: Fixing event log dir to fall back on
storm.local.dir if storm.home is null
Posted by sr...@apache.org.
Fixing event log dir to fall back on storm.local.dir if storm.home is null
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f3baed0d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f3baed0d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f3baed0d
Branch: refs/heads/1.x-branch
Commit: f3baed0d4bb9c415657dfa97f33519994e991e17
Parents: 8d60961
Author: Arun Mahadevan <ai...@hortonworks.com>
Authored: Mon Feb 22 00:03:29 2016 +0530
Committer: Arun Mahadevan <ai...@hortonworks.com>
Committed: Mon Feb 22 00:04:52 2016 +0530
----------------------------------------------------------------------
.../storm/metric/FileBasedEventLogger.java | 22 ++++++++++++++++----
1 file changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f3baed0d/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java b/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
index 076a68c..6a07c0e 100644
--- a/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
+++ b/storm-core/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java
@@ -17,6 +17,7 @@
*/
package org.apache.storm.metric;
+import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
import org.apache.storm.task.TopologyContext;
import org.slf4j.Logger;
@@ -77,11 +78,24 @@ public class FileBasedEventLogger implements IEventLogger {
scheduler.scheduleAtFixedRate(task, FLUSH_INTERVAL_MILLIS, FLUSH_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
}
+ private String getFirstNonNull(String... strings) {
+ for (String str : strings) {
+ if (str != null) {
+ return str;
+ }
+ }
+ return null;
+ }
+
private String getLogDir(Map stormConf) {
- String logDir;
- if ((logDir = System.getProperty("storm.log.dir")) == null
- && (logDir = (String) stormConf.get("storm.log.dir")) == null) {
- logDir = Paths.get(System.getProperty("storm.home"), "logs").toString();
+ String logDir = getFirstNonNull(System.getProperty("storm.log.dir"),
+ (String) stormConf.get("storm.log.dir"));
+ if (logDir == null) {
+ logDir = Paths.get(getFirstNonNull(System.getProperty("storm.home"),
+ System.getProperty("storm.local.dir"),
+ (String) stormConf.get("storm.local.dir"),
+ StringUtils.EMPTY),
+ "logs").toString();
}
return logDir;
}