You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/07/24 17:10:06 UTC
[13/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
STORM-1280 port backtype.storm.daemon.logviewer to java
* address checkstyle violations
* also addressed this from DRPC so far, but needs to decide for DRPC vs Drpc
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6a041a98
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6a041a98
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6a041a98
Branch: refs/heads/master
Commit: 6a041a98fc7fe04db3f520fa283192d96ffb0a56
Parents: 7f84691
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Jul 17 13:35:51 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Jul 17 13:35:51 2017 +0900
----------------------------------------------------------------------
pom.xml | 6 +
storm-buildtools/storm_checkstyle.xml | 6 +
storm-webapp/pom.xml | 2 +-
.../daemon/common/JsonResponseBuilder.java | 14 +-
.../daemon/drpc/webapp/DRPCApplication.java | 2 +-
.../daemon/logviewer/LogviewerConstant.java | 8 +-
.../storm/daemon/logviewer/LogviewerServer.java | 20 ++-
.../handler/LogviewerLogDownloadHandler.java | 30 +++-
.../handler/LogviewerLogPageHandler.java | 116 ++++++++++----
.../handler/LogviewerLogSearchHandler.java | 144 +++++++++++------
.../handler/LogviewerProfileHandler.java | 83 ++++++----
.../logviewer/utils/DirectoryCleaner.java | 51 ++++---
.../daemon/logviewer/utils/LogCleaner.java | 64 ++++----
.../logviewer/utils/LogFileDownloader.java | 18 ++-
.../utils/LogviewerResponseBuilder.java | 59 +++++--
.../logviewer/utils/ResourceAuthorizer.java | 48 ++++--
.../daemon/logviewer/utils/WorkerLogs.java | 82 +++++++---
.../logviewer/webapp/LogviewerApplication.java | 28 ++--
.../logviewer/webapp/LogviewerResource.java | 77 ++++++++--
.../daemon/utils/ListFunctionalSupport.java | 53 ++++++-
.../apache/storm/daemon/utils/StreamUtil.java | 9 ++
.../apache/storm/daemon/utils/URLBuilder.java | 43 ------
.../apache/storm/daemon/utils/UrlBuilder.java | 53 +++++++
.../storm/daemon/drpc/DRPCServerTest.java | 13 +-
.../storm/daemon/logviewer/LogviewerTest.java | 51 -------
.../handler/LogviewerLogPageHandlerTest.java | 32 ++--
.../handler/LogviewerLogSearchHandlerTest.java | 153 ++++++++++---------
.../testsupport/ArgumentsVerifier.java | 15 +-
.../testsupport/MockDirectoryBuilder.java | 12 +-
.../logviewer/testsupport/MockFileBuilder.java | 12 +-
.../daemon/logviewer/utils/LogCleanerTest.java | 61 ++++----
.../logviewer/utils/ResourceAuthorizerTest.java | 24 +--
.../daemon/logviewer/utils/WorkerLogsTest.java | 18 +--
33 files changed, 912 insertions(+), 495 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ffee0cd..9d890d9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -486,6 +486,12 @@
<exclude>**/derby.log</exclude>
<exclude>**/src/dev/**</exclude>
+ <!-- webapp test resources -->
+ <exclude>**/src/test/resources/test-3072.log.test</exclude>
+ <exclude>**/src/test/resources/small-worker.log.test</exclude>
+ <exclude>**/src/test/resources/test-worker.log.test</exclude>
+ <exclude>**/src/test/resources/logviewer-search-context-tests.log.test</exclude>
+
<!-- StormSQL -->
<exclude>**/src/codegen/config.fmpp</exclude>
<exclude>**/src/codegen/data/Parser.tdd</exclude>
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-buildtools/storm_checkstyle.xml
----------------------------------------------------------------------
diff --git a/storm-buildtools/storm_checkstyle.xml b/storm-buildtools/storm_checkstyle.xml
index 248e3ce..3b91be9 100644
--- a/storm-buildtools/storm_checkstyle.xml
+++ b/storm-buildtools/storm_checkstyle.xml
@@ -46,6 +46,9 @@
-->
<module name = "Checker">
+ <!-- Filter out Checkstyle warnings that have been suppressed with the @SuppressWarnings annotation -->
+ <module name="SuppressWarningsFilter" />
+
<property name="charset" value="UTF-8"/>
<property name="severity" value="warning"/>
@@ -58,6 +61,9 @@
</module>
<module name="TreeWalker">
+ <!-- Make the @SuppressWarnings annotations available to Checkstyle -->
+ <module name="SuppressWarningsHolder" />
+
<module name="OuterTypeFilename"/>
<module name="IllegalTokenText">
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/storm-webapp/pom.xml b/storm-webapp/pom.xml
index eeb6ee6..57ccb88 100644
--- a/storm-webapp/pom.xml
+++ b/storm-webapp/pom.xml
@@ -143,7 +143,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>500</maxAllowedViolations>
+ <maxAllowedViolations>4</maxAllowedViolations>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
index 2048d81..dc39bc6 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
@@ -18,12 +18,15 @@
package org.apache.storm.daemon.common;
-import org.apache.storm.ui.UIHelpers;
-
-import javax.ws.rs.core.Response;
import java.util.Collections;
import java.util.Map;
+import javax.ws.rs.core.Response;
+import org.apache.storm.ui.UIHelpers;
+
+/**
+ * Response builder for JSON. It utilizes {@link UIHelpers} to construct JSON body and headers.
+ */
public class JsonResponseBuilder {
private Object data;
private String callback;
@@ -56,6 +59,11 @@ public class JsonResponseBuilder {
return this;
}
+ /**
+ * Build a Response based on given parameters.
+ *
+ * @return A Response.
+ */
public Response build() {
String body = UIHelpers.getJsonResponseBody(data, callback, needSerialize);
Map<String, Object> respHeaders = UIHelpers.getJsonResponseHeaders(callback, headers);
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
index db42310..15cc4d7 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
@@ -24,8 +24,8 @@ import java.util.Set;
import javax.ws.rs.ApplicationPath;
import javax.ws.rs.core.Application;
-import org.apache.storm.daemon.drpc.DRPC;
import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
+import org.apache.storm.daemon.drpc.DRPC;
@ApplicationPath("")
public class DRPCApplication extends Application {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
index 612e0e2..d1a15be 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
@@ -18,6 +18,12 @@
package org.apache.storm.daemon.logviewer;
-public class LogviewerConstant {
+/**
+ * Constants which are used across logviewer related classes.
+ */
+public final class LogviewerConstant {
+ private LogviewerConstant() {
+ }
+
public static final int DEFAULT_BYTES_PER_PAGE = 51200;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
index 1b53809..4f9834d 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
@@ -18,8 +18,17 @@
package org.apache.storm.daemon.logviewer;
+import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES;
+
import com.codahale.metrics.Meter;
import com.google.common.annotations.VisibleForTesting;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
import org.apache.storm.daemon.logviewer.utils.LogCleaner;
@@ -40,14 +49,9 @@ import org.glassfish.jersey.servlet.ServletContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES;
-
+/**
+ * The main entry of Logviewer.
+ */
public class LogviewerServer implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(LogviewerServer.class);
private static final Meter meterShutdownCalls = StormMetricsRegistry.registerMeter("drpc:num-shutdown-calls");
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
index 7529297..7b2fbcb 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
@@ -18,28 +18,52 @@
package org.apache.storm.daemon.logviewer.handler;
+import java.io.IOException;
+import javax.ws.rs.core.Response;
+
import org.apache.storm.daemon.logviewer.utils.LogFileDownloader;
import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-
public class LogviewerLogDownloadHandler {
private WorkerLogs workerLogs;
private final LogFileDownloader logFileDownloadHelper;
+ /**
+ * Constructor.
+ *
+ * @param logRoot root worker log directory
+ * @param daemonLogRoot root daemon log directory
+ * @param workerLogs {@link WorkerLogs}
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, WorkerLogs workerLogs, ResourceAuthorizer resourceAuthorizer) {
this.workerLogs = workerLogs;
this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, resourceAuthorizer);
}
+ /**
+ * Download an worker log.
+ *
+ * @param fileName file to download
+ * @param user username
+ * @return a Response which lets browsers download that file.
+ * @see {@link LogFileDownloader#downloadFile(String, String, boolean)}
+ */
public Response downloadLogFile(String fileName, String user) throws IOException {
workerLogs.setLogFilePermission(fileName);
return logFileDownloadHelper.downloadFile(fileName, user, false);
}
+ /**
+ * Download a daemon log.
+ *
+ * @param fileName file to download
+ * @param user username
+ * @return a Response which lets browsers download that file.
+ * @see {@link LogFileDownloader#downloadFile(String, String, boolean)}
+ */
public Response downloadDaemonLogFile(String fileName, String user) throws IOException {
return logFileDownloadHelper.downloadFile(fileName, user, true);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
index d2488f6..04b1a13 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
@@ -18,22 +18,26 @@
package org.apache.storm.daemon.logviewer.handler;
+import static j2html.TagCreator.a;
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.div;
+import static j2html.TagCreator.form;
+import static j2html.TagCreator.h3;
+import static j2html.TagCreator.head;
+import static j2html.TagCreator.html;
+import static j2html.TagCreator.input;
+import static j2html.TagCreator.link;
+import static j2html.TagCreator.p;
+import static j2html.TagCreator.pre;
+import static j2html.TagCreator.select;
+import static j2html.TagCreator.text;
+import static j2html.TagCreator.title;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+
import j2html.TagCreator;
import j2html.tags.DomContent;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
-import org.apache.storm.daemon.logviewer.LogviewerConstant;
-import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
-import org.apache.storm.daemon.utils.StreamUtil;
-import org.apache.storm.daemon.utils.URLBuilder;
-import org.apache.storm.ui.InvalidRequestException;
-import org.apache.storm.ui.UIHelpers;
-import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ServerUtils;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -54,10 +58,20 @@ import java.util.zip.GZIPInputStream;
import javax.ws.rs.core.Response;
-import static j2html.TagCreator.*;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toList;
-import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.logviewer.LogviewerConstant;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.UrlBuilder;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
public class LogviewerLogPageHandler {
private final String logRoot;
@@ -65,6 +79,14 @@ public class LogviewerLogPageHandler {
private final WorkerLogs workerLogs;
private final ResourceAuthorizer resourceAuthorizer;
+ /**
+ * Constructor.
+ *
+ * @param logRoot root worker log directory
+ * @param daemonLogRoot root daemon log directory
+ * @param workerLogs {@link WorkerLogs}
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerLogPageHandler(String logRoot, String daemonLogRoot,
WorkerLogs workerLogs,
ResourceAuthorizer resourceAuthorizer) {
@@ -74,6 +96,16 @@ public class LogviewerLogPageHandler {
this.resourceAuthorizer = resourceAuthorizer;
}
+ /**
+ * Enumerate worker log files for given criteria.
+ *
+ * @param user username
+ * @param port worker's port, null for all workers
+ * @param topologyId topology ID, null for all topologies
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @return list of worker logs for given criteria
+ */
public Response listLogFiles(String user, Integer port, String topologyId, String callback, String origin) throws IOException {
List<File> fileResults = null;
if (topologyId == null) {
@@ -130,7 +162,18 @@ public class LogviewerLogPageHandler {
return LogviewerResponseBuilder.buildSuccessJsonResponse(files, callback, origin);
}
- public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ /**
+ * Provides a worker log file to view.
+ *
+ * @param fileName file to view
+ * @param start start offset, can be null
+ * @param length length to read in this page, can be null
+ * @param grep search string if request is a result of the search, can be null
+ * @param user username
+ * @return HTML view page of worker log
+ */
+ public Response logPage(String fileName, Integer start, Integer length, String grep, String user)
+ throws IOException, InvalidRequestException {
String rootDir = logRoot;
if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
workerLogs.setLogFilePermission(fileName);
@@ -141,8 +184,6 @@ public class LogviewerLogPageHandler {
File topoDir = file.getParentFile().getParentFile();
if (file.exists() && new File(rootDir).getCanonicalFile().equals(topoDir.getParentFile())) {
- long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
-
SortedSet<File> logFiles;
try {
logFiles = Arrays.stream(topoDir.listFiles())
@@ -169,6 +210,7 @@ public class LogviewerLogPageHandler {
logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
}
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
start = start != null ? start : Long.valueOf(fileLength - length).intValue();
List<DomContent> bodyContents = new ArrayList<>();
@@ -209,15 +251,24 @@ public class LogviewerLogPageHandler {
}
}
- public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ /**
+ * Provides a daemon log file to view.
+ *
+ * @param fileName file to view
+ * @param start start offset, can be null
+ * @param length length to read in this page, can be null
+ * @param grep search string if request is a result of the search, can be null
+ * @param user username
+ * @return HTML view page of daemon log
+ */
+ public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user)
+ throws IOException, InvalidRequestException {
String rootDir = daemonLogRoot;
File file = new File(rootDir, fileName).getCanonicalFile();
String path = file.getCanonicalPath();
boolean isZipFile = path.endsWith(".gz");
if (file.exists() && new File(rootDir).getCanonicalFile().equals(file.getParentFile())) {
- long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
-
// all types of files included
List<File> logFiles = Arrays.stream(new File(rootDir).listFiles())
.filter(File::isFile)
@@ -239,6 +290,7 @@ public class LogviewerLogPageHandler {
logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
}
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
start = start != null ? start : Long.valueOf(fileLength - length).intValue();
List<DomContent> bodyContents = new ArrayList<>();
@@ -331,36 +383,36 @@ public class LogviewerLogPageHandler {
}
private DomContent pagerLinks(String fileName, Integer start, Integer length, Integer fileLength, String type) {
- int prevStart = Math.max(0, start - length);
- int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length;
- List<DomContent> btnLinks = new ArrayList<>();
-
Map<String, Object> urlQueryParams = new HashMap<>();
urlQueryParams.put("file", fileName);
urlQueryParams.put("start", Math.max(0, start - length));
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start));
+ List<DomContent> btnLinks = new ArrayList<>();
+
+ int prevStart = Math.max(0, start - length);
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start));
urlQueryParams.clear();
urlQueryParams.put("file", fileName);
urlQueryParams.put("start", 0);
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "First"));
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "First"));
urlQueryParams.clear();
urlQueryParams.put("file", fileName);
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Last"));
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Last"));
urlQueryParams.clear();
urlQueryParams.put("file", fileName);
urlQueryParams.put("start", Math.min(Math.max(0, fileLength - length), start + length));
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start));
+ int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length;
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start));
return div(btnLinks.toArray(new DomContent[]{}));
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
index 304c2d6..76f1683 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
@@ -18,29 +18,17 @@
package org.apache.storm.daemon.logviewer.handler;
+import static java.util.stream.Collectors.toList;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.first;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.last;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.BooleanUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
-import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.daemon.logviewer.LogviewerConstant;
-import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
-import org.apache.storm.daemon.utils.StreamUtil;
-import org.apache.storm.daemon.utils.URLBuilder;
-import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.ui.InvalidRequestException;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ServerUtils;
-import org.apache.storm.utils.Utils;
-import org.json.simple.JSONAware;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import javax.ws.rs.core.Response;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -58,15 +46,29 @@ import java.util.Map;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
-import static java.util.stream.Collectors.toList;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.first;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.last;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.daemon.logviewer.LogviewerConstant;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.UrlBuilder;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class LogviewerLogSearchHandler {
- private final static Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class);
+ private static final Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class);
public static final int GREP_MAX_SEARCH_SIZE = 1024;
public static final int GREP_BUF_SIZE = 2048;
@@ -79,6 +81,14 @@ public class LogviewerLogSearchHandler {
private final ResourceAuthorizer resourceAuthorizer;
private final Integer logviewerPort;
+ /**
+ * Constructor.
+ *
+ * @param stormConf storm configuration
+ * @param logRoot log root directory
+ * @param daemonLogRoot daemon log root directory
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerLogSearchHandler(Map<String, Object> stormConf, String logRoot, String daemonLogRoot,
ResourceAuthorizer resourceAuthorizer) {
this.stormConf = stormConf;
@@ -89,6 +99,19 @@ public class LogviewerLogSearchHandler {
this.logviewerPort = ObjectReader.getInt(stormConf.get(DaemonConfig.LOGVIEWER_PORT));
}
+ /**
+ * Search from a worker log file.
+ *
+ * @param fileName log file
+ * @param user username
+ * @param isDaemon whether the log file is regarding worker or daemon
+ * @param search search string
+ * @param numMatchesStr the count of maximum matches
+ * @param offsetStr start offset for log file
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @return Response containing JSON content representing search result
+ */
public Response searchLogFile(String fileName, String user, boolean isDaemon, String search,
String numMatchesStr, String offsetStr, String callback, String origin)
throws IOException, InvalidRequestException {
@@ -130,6 +153,21 @@ public class LogviewerLogSearchHandler {
return response;
}
+ /**
+ * Deep search across worker log files in a topology.
+ *
+ * @param topologyId topology ID
+ * @param user username
+ * @param search search string
+ * @param numMatchesStr the count of maximum matches
+ * @param portStr worker port, null or '*' if the request wants to search from all worker logs
+ * @param fileOffsetStr index (offset) of the log files
+ * @param offsetStr start offset for log file
+ * @param searchArchived true if the request wants to search also archived files, false if not
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @return Response containing JSON content representing search result
+ */
public Response deepSearchLogsForTopology(String topologyId, String user, String search,
String numMatchesStr, String portStr, String fileOffsetStr, String offsetStr,
Boolean searchArchived, String callback, String origin) {
@@ -178,8 +216,8 @@ public class LogviewerLogSearchHandler {
if (!containsPort) {
returnValue = new ArrayList<>();
} else {
- File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId +
- Utils.FILE_PATH_SEPARATOR + port);
+ File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId
+ + Utils.FILE_PATH_SEPARATOR + port);
if (!portDir.exists() || logsForPort(user, portDir).isEmpty()) {
returnValue = new ArrayList<>();
@@ -218,22 +256,19 @@ public class LogviewerLogSearchHandler {
}
@VisibleForTesting
- Map<String,Object> substringSearchDaemonLog(File file, String searchString) throws InvalidRequestException {
- return substringSearch(file, searchString, true, 10, 0);
- }
-
- @VisibleForTesting
Map<String,Object> substringSearch(File file, String searchString, int numMatches, int startByteOffset) throws InvalidRequestException {
return substringSearch(file, searchString, false, numMatches, startByteOffset);
}
- private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches, Integer startByteOffset) throws InvalidRequestException {
+ private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches,
+ Integer startByteOffset) throws InvalidRequestException {
try {
if (StringUtils.isEmpty(searchString)) {
throw new IllegalArgumentException("Precondition fails: search string should not be empty.");
}
if (searchString.getBytes("UTF-8").length > GREP_MAX_SEARCH_SIZE) {
- throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + GREP_MAX_SEARCH_SIZE);
+ throw new IllegalArgumentException("Precondition fails: the length of search string should be less than "
+ + GREP_MAX_SEARCH_SIZE);
}
boolean isZipFile = file.getName().endsWith(".gz");
@@ -255,9 +290,8 @@ public class LogviewerLogSearchHandler {
}
ByteBuffer buf = ByteBuffer.allocate(GREP_BUF_SIZE);
- byte[] bufArray = buf.array();
- int totalBytesRead = 0;
- byte[] searchBytes = searchString.getBytes("UTF-8");
+ final byte[] bufArray = buf.array();
+ final byte[] searchBytes = searchString.getBytes("UTF-8");
numMatches = numMatches != null ? numMatches : 10;
startByteOffset = startByteOffset != null ? startByteOffset : 0;
@@ -273,6 +307,7 @@ public class LogviewerLogSearchHandler {
Arrays.fill(bufArray, (byte) 0);
+ int totalBytesRead = 0;
int bytesRead = stream.read(bufArray, 0, Math.min((int) fileLength, GREP_BUF_SIZE));
buf.limit(bytesRead);
totalBytesRead += bytesRead;
@@ -295,7 +330,7 @@ public class LogviewerLogSearchHandler {
// The start index is positioned to find any possible
// occurrence search string that did not quite fit in the
// buffer on the previous read.
- int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length;
+ final int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length;
totalBytesRead = rotateGrepBuffer(buf, stream, totalBytesRead, file, fileLength);
if (totalBytesRead < 0) {
@@ -326,6 +361,10 @@ public class LogviewerLogSearchHandler {
}
}
+ @VisibleForTesting
+ Map<String,Object> substringSearchDaemonLog(File file, String searchString) throws InvalidRequestException {
+ return substringSearch(file, searchString, true, 10, 0);
+ }
/**
* Get the filtered, authorized, sorted log files for a port.
@@ -370,7 +409,7 @@ public class LogviewerLogSearchHandler {
String fileName = WorkerLogs.getTopologyPortWorkerLog(firstLog);
- List<Map<String, Object>> newMatches = new ArrayList<>(matches);
+ final List<Map<String, Object>> newMatches = new ArrayList<>(matches);
Map<String, Object> currentFileMatch = new HashMap<>(theseMatches);
currentFileMatch.put("fileName", fileName);
List<String> splitPath;
@@ -412,7 +451,8 @@ public class LogviewerLogSearchHandler {
private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, File file, int fileLength, int offsetToBuf,
int initBufOffset, BufferedInputStream stream, Integer bytesSkipped,
int bytesRead, ByteBuffer haystack, byte[] needle,
- List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes) throws IOException {
+ List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes)
+ throws IOException {
int bufOffset = initBufOffset;
List<Map<String, Object>> matches = initialMatches;
@@ -422,8 +462,8 @@ public class LogviewerLogSearchHandler {
while (true) {
int offset = offsetOfBytes(haystack.array(), needle, bufOffset);
if (matches.size() < numMatches && offset >= 0) {
- int fileOffset = offsetToBuf + offset;
- int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length;
+ final int fileOffset = offsetToBuf + offset;
+ final int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length;
byte[] beforeArg = null;
byte[] afterArg = null;
@@ -541,7 +581,8 @@ public class LogviewerLogSearchHandler {
* Tries once to read ahead in the stream to fill the context and
* resets the stream to its position before the call.
*/
- private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) throws IOException {
+ private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead)
+ throws IOException {
int numExpected = Math.min(fileLength - bytesRead, GREP_CONTEXT_SIZE);
byte[] afterBytes = new byte[numExpected];
stream.mark(numExpected);
@@ -617,7 +658,7 @@ public class LogviewerLogSearchHandler {
@VisibleForTesting
String urlToMatchCenteredInLogPage(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
- String host = Utils.hostname();
+ final String host = Utils.hostname();
String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 3));
@@ -626,12 +667,12 @@ public class LogviewerLogSearchHandler {
parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE);
- return URLBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
+ return UrlBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
}
@VisibleForTesting
String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
- String host = Utils.hostname();
+ final String host = Utils.hostname();
String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 1));
@@ -640,7 +681,7 @@ public class LogviewerLogSearchHandler {
parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE);
- return URLBuilder.build(String.format("http://%s:%d/api/v1/daemonlog", host, port), parameters);
+ return UrlBuilder.build(String.format("http://%s:%d/api/v1/daemonlog", host, port), parameters);
}
@VisibleForTesting
@@ -651,6 +692,13 @@ public class LogviewerLogSearchHandler {
private String searchString;
private List<Map<String, Object>> matches;
+ /**
+ * Constructor.
+ *
+ * @param fileOffset offset (index) of the files
+ * @param searchString search string
+ * @param matches map representing matched search result
+ */
public Matched(int fileOffset, String searchString, List<Map<String, Object>> matches) {
this.fileOffset = fileOffset;
this.searchString = searchString;
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
index 39608da..d7487e7 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
@@ -18,18 +18,6 @@
package org.apache.storm.daemon.logviewer.handler;
-import j2html.tags.DomContent;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
-import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.utils.ServerUtils;
-
-import javax.ws.rs.core.Response;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
import static j2html.TagCreator.a;
import static j2html.TagCreator.body;
import static j2html.TagCreator.head;
@@ -40,17 +28,44 @@ import static j2html.TagCreator.title;
import static j2html.TagCreator.ul;
import static java.util.stream.Collectors.toList;
+import j2html.tags.DomContent;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.utils.ServerUtils;
+
public class LogviewerProfileHandler {
public static final String WORKER_LOG_FILENAME = "worker.log";
private final String logRoot;
private final ResourceAuthorizer resourceAuthorizer;
+ /**
+ * Constructor.
+ *
+ * @param logRoot worker log root directory
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer) {
this.logRoot = logRoot;
this.resourceAuthorizer = resourceAuthorizer;
}
+ /**
+ * Enumerate dump (profile) files for given worker.
+ *
+ * @param topologyId topology ID
+ * @param hostPort host and port of worker
+ * @param user username
+ * @return The HTML page representing list page of dump files
+ */
public Response listDumpFiles(String topologyId, String hostPort, String user) throws IOException {
String portStr = hostPort.split(":")[1];
File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
@@ -68,6 +83,33 @@ public class LogviewerProfileHandler {
}
}
+ /**
+ * Download a dump file.
+ *
+ * @param topologyId topology ID
+ * @param hostPort host and port of worker
+ * @param fileName dump file name
+ * @param user username
+ * @return a Response which lets browsers download that file.
+ * @see {@link org.apache.storm.daemon.logviewer.utils.LogFileDownloader#downloadFile(String, String, boolean)}
+ */
+ public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException {
+ String portStr = hostPort.split(":")[1];
+ File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
+ File file = new File(dir, fileName);
+
+ if (dir.exists() && file.exists()) {
+ String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
+ if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
+ return LogviewerResponseBuilder.buildDownloadFile(file);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
private String buildDumpFileListPage(String topologyId, String hostPort, File dir) throws IOException {
List<DomContent> liTags = getProfilerDumpFiles(dir).stream()
.map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + file)))
@@ -86,23 +128,6 @@ public class LogviewerProfileHandler {
).render();
}
- public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException {
- String portStr = hostPort.split(":")[1];
- File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
- File file = new File(dir, fileName);
-
- if (dir.exists() && file.exists()) {
- String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
- if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
- return LogviewerResponseBuilder.buildDownloadFile(file);
- } else {
- return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
- }
- } else {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- }
- }
-
private List<String> getProfilerDumpFiles(File dir) throws IOException {
List<File> filesForDir = DirectoryCleaner.getFilesForDir(dir);
return filesForDir.stream().filter(file -> {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
index c8a4f69..7bbb8d7 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
@@ -15,19 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.logviewer.utils;
-import java.io.IOException;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.DirectoryStream;
-import java.util.Stack;
import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
import java.util.Comparator;
+import java.util.List;
import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.Stack;
import java.util.regex.Pattern;
import org.slf4j.Logger;
@@ -37,25 +38,35 @@ import org.slf4j.LoggerFactory;
* Provide methods to help Logviewer to clean up
* files in directories and to get a list of files without
* worrying about excessive memory usage.
- *
*/
public class DirectoryCleaner {
private static final Logger LOG = LoggerFactory.getLogger(DirectoryCleaner.class);
// used to recognize the pattern of active log files, we may remove the "current" from this list
private static final Pattern ACTIVE_LOG_PATTERN = Pattern.compile(".*\\.(log|err|out|current|yaml|pid)$");
// used to recognize the pattern of some meta files in a worker log directory
- private static final Pattern META_LOG_PATTERN= Pattern.compile(".*\\.(yaml|pid)$");
+ private static final Pattern META_LOG_PATTERN = Pattern.compile(".*\\.(yaml|pid)$");// max number of files to delete for every round
+
+ private static final int PQ_SIZE = 1024;
+ private static final int MAX_ROUNDS = 512; // max rounds of scanning the dirs
+ public static final int MAX_NUMBER_OF_FILES_FOR_DIR = 1024;
// not defining this as static is to allow for mocking in tests
+
+ /**
+ * Creates DirectoryStream for give directory.
+ *
+ * @param dir File instance representing specific directory
+ * @return DirectoryStream
+ */
public DirectoryStream<Path> getStreamForDirectory(File dir) throws IOException {
- DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath());
- return stream;
+ return Files.newDirectoryStream(dir.toPath());
}
/**
* If totalSize of files exceeds the either the per-worker quota or global quota,
* Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs.
* We use the parameter forPerDir to switch between the two deletion modes.
+ *
* @param dirs the list of directories to be scanned for deletion
* @param quota the per-dir quota or the total quota for the all directories
* @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories globally
@@ -64,8 +75,6 @@ public class DirectoryCleaner {
*/
public int deleteOldestWhileTooLarge(List<File> dirs,
long quota, boolean forPerDir, Set<String> activeDirs) throws IOException {
- final int PQ_SIZE = 1024; // max number of files to delete for every round
- final int MAX_ROUNDS = 512; // max rounds of scanning the dirs
long totalSize = 0;
int deletedFiles = 0;
@@ -133,12 +142,12 @@ public class DirectoryCleaner {
round++;
if (round >= MAX_ROUNDS) {
if (forPerDir) {
- LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " +
- "a single directory : {}, will delete the rest files in next interval.",
+ LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in "
+ + "a single directory : {}, will delete the rest files in next interval.",
MAX_ROUNDS, dirs.get(0).getCanonicalPath());
} else {
- LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " +
- "will delete the rest files in next interval.", MAX_ROUNDS);
+ LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, "
+ + "will delete the rest files in next interval.", MAX_ROUNDS);
}
break;
}
@@ -165,15 +174,19 @@ public class DirectoryCleaner {
return false;
}
- // Note that to avoid memory problem, we only return the first 1024 files in a directory
+ /**
+ * Lists files in directory.
+ * Note that to avoid memory problem, we only return the first 1024 files in a directory.
+ *
+ * @param dir directory to get file list
+ * @return files in directory
+ */
public static List<File> getFilesForDir(File dir) throws IOException {
List<File> files = new ArrayList<File>();
- final int MAX_NUM = 1024;
-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath())) {
for (Path path : stream) {
files.add(path.toFile());
- if (files.size() >= MAX_NUM) {
+ if (files.size() >= MAX_NUMBER_OF_FILES_FOR_DIR) {
break;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
index 6e4e073..bfd80a3 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
@@ -18,16 +18,16 @@
package org.apache.storm.daemon.logviewer.utils;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
+
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB;
+
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
-import org.apache.storm.StormTimer;
-import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
@@ -41,23 +41,24 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.function.BinaryOperator;
import java.util.stream.StreamSupport;
-import static java.util.stream.Collectors.joining;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toList;
-import static java.util.stream.Collectors.toMap;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB;
+import org.apache.commons.io.IOUtils;
+import org.apache.storm.StormTimer;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * Cleans dead workers logs and directories.
+ */
public class LogCleaner implements Runnable, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
@@ -71,6 +72,14 @@ public class LogCleaner implements Runnable, Closeable {
private final long maxSumWorkerLogsSizeMb;
private long maxPerWorkerLogsSizeMb;
+ /**
+ * Constuctor.
+ *
+ * @param stormConf configuration map for Storm cluster
+ * @param workerLogs {@link WorkerLogs} instance
+ * @param directoryCleaner {@link DirectoryCleaner} instance
+ * @param logRootDir root log directory
+ */
public LogCleaner(Map<String, Object> stormConf, WorkerLogs workerLogs, DirectoryCleaner directoryCleaner,
File logRootDir) {
this.stormConf = stormConf;
@@ -87,6 +96,9 @@ public class LogCleaner implements Runnable, Closeable {
maxSumWorkerLogsSizeMb, maxPerWorkerLogsSizeMb);
}
+ /**
+ * Start log cleanup thread.
+ */
public void start() {
if (intervalSecs != null) {
LOG.debug("starting log cleanup thread at interval: {}", intervalSecs);
@@ -100,9 +112,9 @@ public class LogCleaner implements Runnable, Closeable {
} else {
LOG.warn("The interval for log cleanup is not set. Skip starting log cleanup thread.");
}
-
}
+ @Override
public void close() {
if (logviewerCleanupTimer != null) {
try {
@@ -116,6 +128,7 @@ public class LogCleaner implements Runnable, Closeable {
/**
* Delete old log dirs for which the workers are no longer alive.
*/
+ @Override
public void run() {
try {
int nowSecs = Time.currentTimeSecs();
@@ -152,7 +165,8 @@ public class LogCleaner implements Runnable, Closeable {
@VisibleForTesting
List<Integer> perWorkerDirCleanup(long size) {
return workerLogs.getAllWorkerDirs().stream()
- .map(Unchecked.function(dir -> directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null)))
+ .map(Unchecked.function(dir ->
+ directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null)))
.collect(toList());
}
@@ -167,14 +181,6 @@ public class LogCleaner implements Runnable, Closeable {
return directoryCleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs);
}
- private Set<String> getAliveIds(int nowSecs) throws Exception {
- return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
- .filter(entry -> Objects.nonNull(entry.getValue())
- && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf))
- .map(Map.Entry::getKey)
- .collect(toCollection(TreeSet::new));
- }
-
/**
* Delete the topo dir if it contains zero port dirs.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
index 7d9cb50..62c08da 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
@@ -18,22 +18,38 @@
package org.apache.storm.daemon.logviewer.utils;
-import javax.ws.rs.core.Response;
import java.io.File;
import java.io.IOException;
+import javax.ws.rs.core.Response;
+
public class LogFileDownloader {
private final String logRoot;
private final String daemonLogRoot;
private final ResourceAuthorizer resourceAuthorizer;
+ /**
+ * Constructor.
+ *
+ * @param logRoot root worker log directory
+ * @param daemonLogRoot root daemon log directory
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogFileDownloader(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
this.logRoot = logRoot;
this.daemonLogRoot = daemonLogRoot;
this.resourceAuthorizer = resourceAuthorizer;
}
+ /**
+ * Checks authorization for the log file and download
+ *
+ * @param fileName file to download
+ * @param user username
+ * @param isDaemon true if the file is a daemon log, false if the file is an worker log
+ * @return a Response which lets browsers download that file.
+ */
public Response downloadFile(String fileName, String user, boolean isDaemon) throws IOException {
String rootDir = isDaemon ? daemonLogRoot : logRoot;
File file = new File(rootDir, fileName).getCanonicalFile();
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
index d0f2857..59570ee 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
@@ -18,13 +18,12 @@
package org.apache.storm.daemon.logviewer.utils;
-import com.google.common.io.ByteStreams;
-import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.ui.UIHelpers;
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.h2;
+import static javax.ws.rs.core.Response.Status.OK;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
+import com.google.common.io.ByteStreams;
import java.io.BufferedOutputStream;
import java.io.File;
@@ -35,26 +34,46 @@ import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
-import static j2html.TagCreator.body;
-import static j2html.TagCreator.h2;
-import static javax.ws.rs.core.Response.Status.OK;
-import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.ui.UIHelpers;
public class LogviewerResponseBuilder {
private LogviewerResponseBuilder() {
}
+ /**
+ * Build a Response object representing success response with HTML entity.
+ *
+ * @param content HTML entity content, String type
+ */
public static Response buildSuccessHtmlResponse(String content) {
return Response.status(OK).entity(content)
.type(MediaType.TEXT_HTML_TYPE).build();
}
+ /**
+ * Build a Response object representing success response with JSON entity.
+ *
+ * @param entity entity object to represent it as JSON
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @see {@link JsonResponseBuilder}
+ */
public static Response buildSuccessJsonResponse(Object entity, String callback, String origin) {
return new JsonResponseBuilder().setData(entity).setCallback(callback)
.setHeaders(LogviewerResponseBuilder.getHeadersForSuccessResponse(origin)).build();
}
+ /**
+ * Build a Response object representing download a file.
+ *
+ * @param file file to download
+ */
public static Response buildDownloadFile(File file) throws IOException {
// do not close this InputStream in method: it will be used from jetty server
InputStream is = new FileInputStream(file);
@@ -65,6 +84,11 @@ public class LogviewerResponseBuilder {
.build();
}
+ /**
+ * Build a Response object representing unauthorized user, with HTML response.
+ *
+ * @param user username
+ */
public static Response buildResponseUnautohrizedUser(String user) {
String entity = buildUnauthorizedUserHtml(user);
return Response.status(OK)
@@ -73,6 +97,9 @@ public class LogviewerResponseBuilder {
.build();
}
+ /**
+ * Build a Response object representing page not found.
+ */
public static Response buildResponsePageNotFound() {
return Response.status(404)
.entity("Page not found")
@@ -80,11 +107,23 @@ public class LogviewerResponseBuilder {
.build();
}
+ /**
+ * Build a Response object representing unauthorized user, with JSON response.
+ *
+ * @param user username
+ * @param callback callback for JSONP
+ */
public static Response buildUnauthorizedUserJsonResponse(String user, String callback) {
return new JsonResponseBuilder().setData(UIHelpers.unauthorizedUserJson(user))
.setCallback(callback).setStatus(401).build();
}
+ /**
+ * Build a Response object representing exception, with JSON response.
+ *
+ * @param ex Exception object
+ * @param callback callback for JSONP
+ */
public static Response buildExceptionJsonResponse(Exception ex, String callback) {
return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(ex))
.setCallback(callback).setStatus(500).build();
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
index 55691b2..16f408a 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
@@ -18,16 +18,10 @@
package org.apache.storm.daemon.logviewer.utils;
+import static org.apache.storm.DaemonConfig.UI_FILTER;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.Config;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.security.auth.AuthUtils;
-import org.apache.storm.security.auth.IGroupMappingServiceProvider;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ServerConfigUtils;
-import org.apache.storm.utils.Utils;
import java.io.File;
import java.io.IOException;
@@ -37,28 +31,51 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import static org.apache.storm.DaemonConfig.UI_FILTER;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.security.auth.AuthUtils;
+import org.apache.storm.security.auth.IGroupMappingServiceProvider;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerConfigUtils;
+import org.apache.storm.utils.Utils;
public class ResourceAuthorizer {
private final Map<String, Object> stormConf;
private final IGroupMappingServiceProvider groupMappingServiceProvider;
+ /**
+ * Constuctor.
+ *
+ * @param stormConf storm configuration
+ */
public ResourceAuthorizer(Map<String, Object> stormConf) {
this.stormConf = stormConf;
this.groupMappingServiceProvider = AuthUtils.GetGroupMappingServiceProviderPlugin(stormConf);
}
+ /**
+ * Checks whether user is allowed to access file via UI. Always true when UI filter is not set.
+ *
+ * @param fileName file name to access
+ * @param user username
+ */
public boolean isUserAllowedToAccessFile(String fileName, String user) {
return isUiFilterNotSet() || isAuthorizedLogUser(user, fileName);
}
+ /**
+ * Checks whether user is authorized to access file. Checks regardless of UI filter.
+ *
+ * @param user username
+ * @param fileName file name to access
+ */
public boolean isAuthorizedLogUser(String user, String fileName) {
if (StringUtils.isEmpty(user) || StringUtils.isEmpty(fileName)
|| getLogUserGroupWhitelist(fileName) == null) {
return false;
} else {
- Set<String> groups = getUserGroups(user);
LogUserGroupWhitelist whitelist = getLogUserGroupWhitelist(fileName);
List<String> logsUsers = new ArrayList<>();
@@ -70,11 +87,18 @@ public class ResourceAuthorizer {
logsGroups.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_GROUPS)));
logsGroups.addAll(whitelist.getGroupWhitelist());
- return logsUsers.stream().anyMatch(u -> u.equals(user)) ||
- Sets.intersection(groups, new HashSet<>(logsGroups)).size() > 0;
+ Set<String> groups = getUserGroups(user);
+
+ return logsUsers.stream().anyMatch(u -> u.equals(user))
+ || Sets.intersection(groups, new HashSet<>(logsGroups)).size() > 0;
}
}
+ /**
+ * Get the whitelist of users and groups for given file.
+ *
+ * @param fileName file name to get the whitelist
+ */
public LogUserGroupWhitelist getLogUserGroupWhitelist(String fileName) {
File wlFile = ServerConfigUtils.getLogMetaDataFile(fileName);
Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(wlFile.getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
index 6018b79..eba6404 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
@@ -18,16 +18,13 @@
package org.apache.storm.daemon.logviewer.utils;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.storm.Config.SUPERVISOR_RUN_WORKER_AS_USER;
+import static org.apache.storm.Config.TOPOLOGY_SUBMITTER_USER;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+
import com.google.common.collect.Lists;
-import org.apache.storm.daemon.supervisor.ClientSupervisorUtils;
-import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
-import org.jooq.lambda.tuple.Tuple2;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -43,12 +40,19 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.stream.Stream;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toMap;
-import static org.apache.storm.Config.SUPERVISOR_RUN_WORKER_AS_USER;
-import static org.apache.storm.Config.TOPOLOGY_SUBMITTER_USER;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+import org.apache.storm.daemon.supervisor.ClientSupervisorUtils;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+import org.jooq.lambda.tuple.Tuple2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * A class that knows about how to operate with worker log directory.
+ */
public class WorkerLogs {
private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
@@ -56,19 +60,30 @@ public class WorkerLogs {
private final Map<String, Object> stormConf;
private final File logRootDir;
+ /**
+ * Constructor.
+ *
+ * @param stormConf storm configuration
+ * @param logRootDir the log root directory
+ */
public WorkerLogs(Map<String, Object> stormConf, File logRootDir) {
this.stormConf = stormConf;
this.logRootDir = logRootDir;
}
+ /**
+ * Set permission of log file so that logviewer can serve the file.
+ *
+ * @param fileName log file
+ */
public void setLogFilePermission(String fileName) throws IOException {
File file = new File(logRootDir, fileName).getCanonicalFile();
boolean runAsUser = ObjectReader.getBoolean(stormConf.get(SUPERVISOR_RUN_WORKER_AS_USER), false);
File parent = new File(logRootDir, fileName).getParentFile();
Optional<File> mdFile = (parent == null) ? Optional.empty() : getMetadataFileForWorkerLogDir(parent);
- Optional<String> topoOwner = mdFile.isPresent() ?
- Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().getCanonicalPath())) :
- Optional.empty();
+ Optional<String> topoOwner = mdFile.isPresent()
+ ? Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().getCanonicalPath()))
+ : Optional.empty();
if (runAsUser && topoOwner.isPresent() && file.exists() && !Files.isReadable(file.toPath())) {
LOG.debug("Setting permissions on file {} with topo-owner {}", fileName, topoOwner);
@@ -78,6 +93,9 @@ public class WorkerLogs {
}
}
+ /**
+ * Return a list of all log files from worker directories in root log directory.
+ */
public List<File> getAllLogsForRootDir() throws IOException {
List<File> files = new ArrayList<>();
Set<File> topoDirFiles = getAllWorkerDirs();
@@ -90,6 +108,9 @@ public class WorkerLogs {
return files;
}
+ /**
+ * Return a set of all worker directories in root log directory.
+ */
public Set<File> getAllWorkerDirs() {
File[] rootDirFiles = logRootDir.listFiles();
if (rootDirFiles != null) {
@@ -116,6 +137,10 @@ public class WorkerLogs {
.collect(toCollection(TreeSet::new));
}
+ /**
+ * Return a metadata file (worker.yaml) for given worker log directory.
+ * @param logDir worker log directory
+ */
public Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
File metaFile = new File(logDir, WORKER_YAML);
if (metaFile.exists()) {
@@ -126,16 +151,31 @@ public class WorkerLogs {
}
}
+ /**
+ * Return worker id from worker meta file.
+ *
+ * @param metaFile metadata file
+ */
public String getWorkerIdFromMetadataFile(String metaFile) {
Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
return ObjectReader.getString(map.get("worker-id"), null);
}
+ /**
+ * Return topology owner from worker meta file.
+ *
+ * @param metaFile metadata file
+ */
public String getTopologyOwnerFromMetadataFile(String metaFile) {
Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
return ObjectReader.getString(map.get(TOPOLOGY_SUBMITTER_USER), null);
}
+ /**
+ * Retrieve the set of alive worker IDs.
+ *
+ * @param nowSecs current time in seconds
+ */
public Set<String> getAliveIds(int nowSecs) throws Exception {
return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
.filter(entry -> Objects.nonNull(entry.getValue())
@@ -144,6 +184,12 @@ public class WorkerLogs {
.collect(toCollection(TreeSet::new));
}
+ /**
+ * Finds a worker ID for each directory in set and return it as map.
+ *
+ * @param logDirs directories to check whether they're worker directories or not
+ * @return the pair of worker ID, directory. worker ID will be an empty string if the directory is not a worker directory.
+ */
public Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
// we could also make this static, but not to do it due to mock
return logDirs.stream().map(Unchecked.function(logDir -> {
@@ -156,6 +202,8 @@ public class WorkerLogs {
/**
* Return the path of the worker log with the format of topoId/port/worker.log.*
+ *
+ * @param file worker log
*/
public static String getTopologyPortWorkerLog(File file) {
try {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
index 32a723e..c44e8cd 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
@@ -18,31 +18,32 @@
package org.apache.storm.daemon.logviewer.webapp;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME;
+
+import java.io.File;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.ApplicationPath;
+import javax.ws.rs.core.Application;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.Appender;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.RollingFileAppender;
import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
-import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
import org.apache.storm.security.auth.AuthUtils;
import org.apache.storm.security.auth.IHttpCredentialsPlugin;
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.ObjectReader;
-import javax.ws.rs.ApplicationPath;
-import javax.ws.rs.core.Application;
-import java.io.File;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME;
-
@ApplicationPath("")
public class LogviewerApplication extends Application {
private static Map<String, Object> stormConf;
@@ -75,6 +76,11 @@ public class LogviewerApplication extends Application {
return singletons;
}
+ /**
+ * Spot to inject storm configuration before initializing LogviewerApplication instance.
+ *
+ * @param stormConf storm configuration
+ */
public static void setup(Map<String, Object> stormConf) {
LogviewerApplication.stormConf = stormConf;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
index bc6529c..478001d 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
@@ -19,13 +19,25 @@
package org.apache.storm.daemon.logviewer.webapp;
import com.codahale.metrics.Meter;
+
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
-import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
import org.apache.storm.metric.StormMetricsRegistry;
import org.apache.storm.security.auth.IHttpCredentialsPlugin;
import org.apache.storm.ui.InvalidRequestException;
@@ -33,24 +45,20 @@ import org.apache.storm.ui.UIHelpers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.net.URLDecoder;
-import java.util.Map;
-
+/**
+ * Handles HTTP requests for Logviewer.
+ */
@Path("/")
public class LogviewerResource {
private static final Logger LOG = LoggerFactory.getLogger(LogviewerResource.class);
private static final Meter meterLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-log-page-http-requests");
- private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-daemonlog-page-http-requests");
- private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-file-http-requests");
- private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-daemon-file-http-requests");
+ private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter(
+ "logviewer:num-daemonlog-page-http-requests");
+ private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter(
+ "logviewer:num-download-log-file-http-requests");
+ private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter(
+ "logviewer:num-download-log-daemon-file-http-requests");
private static final Meter meterListLogsHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-list-logs-http-requests");
private final LogviewerLogPageHandler logviewer;
@@ -59,6 +67,15 @@ public class LogviewerResource {
private final LogviewerLogSearchHandler logSearchHandler;
private final IHttpCredentialsPlugin httpCredsHandler;
+ /**
+ * Constructor.
+ *
+ * @param logviewerParam {@link LogviewerLogPageHandler}
+ * @param profileHandler {@link LogviewerProfileHandler}
+ * @param logDownloadHandler {@link LogviewerLogDownloadHandler}
+ * @param logSearchHandler {@link LogviewerLogSearchHandler}
+ * @param httpCredsHandler {@link IHttpCredentialsPlugin}
+ */
public LogviewerResource(LogviewerLogPageHandler logviewerParam, LogviewerProfileHandler profileHandler,
LogviewerLogDownloadHandler logDownloadHandler, LogviewerLogSearchHandler logSearchHandler,
IHttpCredentialsPlugin httpCredsHandler) {
@@ -69,6 +86,9 @@ public class LogviewerResource {
this.httpCredsHandler = httpCredsHandler;
}
+ /**
+ * Handles '/log' request.
+ */
@GET
@Path("/log")
public Response log(@Context HttpServletRequest request) throws IOException {
@@ -87,6 +107,9 @@ public class LogviewerResource {
}
}
+ /**
+ * Handles '/daemonlog' request.
+ */
@GET
@Path("/daemonlog")
public Response daemonLog(@Context HttpServletRequest request) throws IOException {
@@ -105,6 +128,9 @@ public class LogviewerResource {
}
}
+ /**
+ * Handles '/searchLogs' request.
+ */
@GET
@Path("/searchLogs")
public Response searchLogs(@Context HttpServletRequest request) throws IOException {
@@ -117,6 +143,9 @@ public class LogviewerResource {
return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
}
+ /**
+ * Handles '/listLogs' request.
+ */
@GET
@Path("/listLogs")
public Response listLogs(@Context HttpServletRequest request) throws IOException {
@@ -131,6 +160,9 @@ public class LogviewerResource {
return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
}
+ /**
+ * Handles '/dumps' (listing dump files) request.
+ */
@GET
@Path("/dumps/{topo-id}/{host-port}")
public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
@@ -139,6 +171,9 @@ public class LogviewerResource {
return profileHandler.listDumpFiles(topologyId, hostPort, user);
}
+ /**
+ * Handles '/dumps' (downloading specific dump file) request.
+ */
@GET
@Path("/dumps/{topo-id}/{host-port}/{filename}")
public Response downloadDumpFile(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
@@ -147,6 +182,9 @@ public class LogviewerResource {
return profileHandler.downloadDumpFile(topologyId, hostPort, fileName, user);
}
+ /**
+ * Handles '/download' (downloading specific log file) request.
+ */
@GET
@Path("/download")
public Response downloadLogFile(@Context HttpServletRequest request) throws IOException {
@@ -158,6 +196,9 @@ public class LogviewerResource {
return logDownloadHandler.downloadLogFile(decodedFileName, user);
}
+ /**
+ * Handles '/daemondownload' (downloading specific daemon log file) request.
+ */
@GET
@Path("/daemondownload")
public Response downloadDaemonLogFile(@Context HttpServletRequest request) throws IOException {
@@ -169,6 +210,9 @@ public class LogviewerResource {
return logDownloadHandler.downloadDaemonLogFile(decodedFileName, user);
}
+ /**
+ * Handles '/search' (searching from specific worker or daemon log file) request.
+ */
@GET
@Path("/search/{file}")
public Response search(@PathParam("file") String file, @Context HttpServletRequest request) throws IOException {
@@ -191,6 +235,9 @@ public class LogviewerResource {
}
}
+ /**
+ * Handles '/deepSearch' request.
+ */
@GET
@Path("/deepSearch/{topoId}")
public Response deepSearch(@PathParam("topoId") String topologyId,