You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/07/24 17:09:54 UTC
[01/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Repository: storm
Updated Branches:
refs/heads/master 1622f19c0 -> 3c53dd516
STORM-1280 port backtype.storm.daemon.logviewer to java
* apply recent change from logviewer.clj (0201ae7)
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bb5c247f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bb5c247f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bb5c247f
Branch: refs/heads/master
Commit: bb5c247f9ba595ec0e9dc88284da568011520aff
Parents: 44b268b
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Jul 14 11:56:28 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Jul 14 12:11:41 2017 +0900
----------------------------------------------------------------------
.../storm/daemon/logviewer/LogviewerServer.java | 8 +-
.../handler/LogviewerLogDownloadHandler.java | 6 +-
.../handler/LogviewerLogPageHandler.java | 7 +-
.../daemon/logviewer/utils/LogCleaner.java | 82 +++--------
.../daemon/logviewer/utils/WorkerLogs.java | 141 +++++++++++++++++--
.../logviewer/webapp/LogviewerApplication.java | 6 +-
.../handler/LogviewerLogPageHandlerTest.java | 6 +-
.../daemon/logviewer/utils/LogCleanerTest.java | 81 ++++-------
.../daemon/logviewer/utils/WorkerLogsTest.java | 75 ++++++++++
9 files changed, 274 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
index 2b26702..1b53809 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
@@ -23,10 +23,12 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
import org.apache.storm.daemon.logviewer.utils.LogCleaner;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
import org.apache.storm.daemon.logviewer.webapp.LogviewerApplication;
import org.apache.storm.metric.StormMetricsRegistry;
import org.apache.storm.ui.FilterConfiguration;
import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.Utils;
import org.eclipse.jetty.server.Server;
@@ -38,6 +40,7 @@ import org.glassfish.jersey.servlet.ServletContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@@ -158,10 +161,13 @@ public class LogviewerServer implements AutoCloseable {
Utils.setupDefaultUncaughtExceptionHandler();
Map<String, Object> conf = Utils.readStormConfig();
+ String logRoot = ConfigUtils.workerArtifactsRoot(conf);
+ File logRootFile = new File(logRoot);
+ WorkerLogs workerLogs = new WorkerLogs(conf, logRootFile);
DirectoryCleaner directoryCleaner = new DirectoryCleaner();
try (LogviewerServer server = new LogviewerServer(conf);
- LogCleaner logCleaner = new LogCleaner(conf, directoryCleaner)) {
+ LogCleaner logCleaner = new LogCleaner(conf, workerLogs, directoryCleaner, logRootFile)) {
Utils.addShutdownHookWithForceKillIn1Sec(() -> server.close());
logCleaner.start();
StormMetricsRegistry.startMetricsReporters(conf);
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
index b8f32f2..7529297 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
@@ -20,19 +20,23 @@ package org.apache.storm.daemon.logviewer.handler;
import org.apache.storm.daemon.logviewer.utils.LogFileDownloader;
import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
import javax.ws.rs.core.Response;
import java.io.IOException;
public class LogviewerLogDownloadHandler {
+ private WorkerLogs workerLogs;
private final LogFileDownloader logFileDownloadHelper;
- public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
+ public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, WorkerLogs workerLogs, ResourceAuthorizer resourceAuthorizer) {
+ this.workerLogs = workerLogs;
this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, resourceAuthorizer);
}
public Response downloadLogFile(String fileName, String user) throws IOException {
+ workerLogs.setLogFilePermission(fileName);
return logFileDownloadHelper.downloadFile(fileName, user, false);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
index 0881b69..d2488f6 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
@@ -62,12 +62,15 @@ import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
public class LogviewerLogPageHandler {
private final String logRoot;
private final String daemonLogRoot;
+ private final WorkerLogs workerLogs;
private final ResourceAuthorizer resourceAuthorizer;
public LogviewerLogPageHandler(String logRoot, String daemonLogRoot,
+ WorkerLogs workerLogs,
ResourceAuthorizer resourceAuthorizer) {
this.logRoot = logRoot;
this.daemonLogRoot = daemonLogRoot;
+ this.workerLogs = workerLogs;
this.resourceAuthorizer = resourceAuthorizer;
}
@@ -75,7 +78,7 @@ public class LogviewerLogPageHandler {
List<File> fileResults = null;
if (topologyId == null) {
if (port == null) {
- fileResults = WorkerLogs.getAllLogsForRootDir(new File(logRoot));
+ fileResults = workerLogs.getAllLogsForRootDir();
} else {
fileResults = new ArrayList<>();
@@ -130,6 +133,8 @@ public class LogviewerLogPageHandler {
public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
String rootDir = logRoot;
if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ workerLogs.setLogFilePermission(fileName);
+
File file = new File(rootDir, fileName).getCanonicalFile();
String path = file.getCanonicalPath();
boolean isZipFile = path.endsWith(".gz");
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
index cca2a52..6e4e073 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
@@ -22,12 +22,10 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
import org.apache.storm.StormTimer;
import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Utils;
import org.jooq.lambda.Unchecked;
-import org.jooq.lambda.tuple.Tuple2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,23 +60,23 @@ import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_M
public class LogCleaner implements Runnable, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
- public static final String WORKER_YAML = "worker.yaml";
private final Map<String, Object> stormConf;
private final Integer intervalSecs;
- private final String logRootDir;
+ private final File logRootDir;
private final DirectoryCleaner directoryCleaner;
+ private final WorkerLogs workerLogs;
private StormTimer logviewerCleanupTimer;
private final long maxSumWorkerLogsSizeMb;
private long maxPerWorkerLogsSizeMb;
- public LogCleaner(Map<String, Object> stormConf, DirectoryCleaner directoryCleaner) {
- String logRootDir = ConfigUtils.workerArtifactsRoot(stormConf);
-
+ public LogCleaner(Map<String, Object> stormConf, WorkerLogs workerLogs, DirectoryCleaner directoryCleaner,
+ File logRootDir) {
this.stormConf = stormConf;
this.intervalSecs = ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
this.logRootDir = logRootDir;
+ this.workerLogs = workerLogs;
this.directoryCleaner = directoryCleaner;
maxSumWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB));
@@ -121,7 +119,7 @@ public class LogCleaner implements Runnable, Closeable {
public void run() {
try {
int nowSecs = Time.currentTimeSecs();
- Set<File> oldLogDirs = selectDirsForCleanup(nowSecs * 1000, logRootDir);
+ Set<File> oldLogDirs = selectDirsForCleanup(nowSecs * 1000);
SortedSet<File> deadWorkerDirs = getDeadWorkerDirs(nowSecs, oldLogDirs);
@@ -141,8 +139,8 @@ public class LogCleaner implements Runnable, Closeable {
}
}));
- perWorkerDirCleanup(new File(logRootDir), maxPerWorkerLogsSizeMb * 1024 * 1024, directoryCleaner);
- globalLogCleanup(new File(logRootDir), maxSumWorkerLogsSizeMb * 1024 * 1024, directoryCleaner);
+ perWorkerDirCleanup(maxPerWorkerLogsSizeMb * 1024 * 1024);
+ globalLogCleanup(maxSumWorkerLogsSizeMb * 1024 * 1024);
} catch (Exception ex) {
LOG.error("Exception while cleaning up old log.", ex);
}
@@ -152,9 +150,9 @@ public class LogCleaner implements Runnable, Closeable {
* Delete the oldest files in each overloaded worker log dir.
*/
@VisibleForTesting
- List<Integer> perWorkerDirCleanup(File rootDir, long size, DirectoryCleaner cleaner) {
- return WorkerLogs.getAllWorkerDirs(rootDir).stream()
- .map(Unchecked.function(dir -> cleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null)))
+ List<Integer> perWorkerDirCleanup(long size) {
+ return workerLogs.getAllWorkerDirs().stream()
+ .map(Unchecked.function(dir -> directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null)))
.collect(toList());
}
@@ -162,53 +160,11 @@ public class LogCleaner implements Runnable, Closeable {
* Delete the oldest files in overloaded worker-artifacts globally.
*/
@VisibleForTesting
- int globalLogCleanup(File rootDir, long size, DirectoryCleaner cleaner) throws Exception {
- List<File> workerDirs = new ArrayList<>(WorkerLogs.getAllWorkerDirs(rootDir));
- Set<String> aliveWorkerDirs = new HashSet<>(getAliveWorkerDirs(rootDir));
-
- return cleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs);
- }
-
- /**
- * Return a sorted set of java.io.Files that were written by workers that are now active.
- */
- @VisibleForTesting
- SortedSet<String> getAliveWorkerDirs(File rootDir) throws Exception {
- Set<String> aliveIds = getAliveIds(Time.currentTimeSecs());
- Set<File> logDirs = WorkerLogs.getAllWorkerDirs(rootDir);
- Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
-
- return idToDir.entrySet().stream()
- .filter(entry -> aliveIds.contains(entry.getKey()))
- .map(Unchecked.function(entry -> entry.getValue().getCanonicalPath()))
- .collect(toCollection(TreeSet::new));
- }
+ int globalLogCleanup(long size) throws Exception {
+ List<File> workerDirs = new ArrayList<>(workerLogs.getAllWorkerDirs());
+ Set<String> aliveWorkerDirs = new HashSet<>(workerLogs.getAliveWorkerDirs());
- @VisibleForTesting
- Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
- return logDirs.stream().map(Unchecked.function(logDir -> {
- Optional<File> metaFile = getMetadataFileForWorkerLogDir(logDir);
-
- return metaFile.map(Unchecked.function(m -> new Tuple2<>(getWorkerIdFromMetadataFile(m.getCanonicalPath()), logDir)))
- .orElse(new Tuple2<>("", logDir));
- })).collect(toMap(Tuple2::v1, Tuple2::v2));
- }
-
- @VisibleForTesting
- Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
- File metaFile = new File(logDir, WORKER_YAML);
- if (metaFile.exists()) {
- return Optional.of(metaFile);
- } else {
- LOG.warn("Could not find {} to clean up for {}", metaFile.getCanonicalPath(), logDir);
- return Optional.empty();
- }
- }
-
- @VisibleForTesting
- String getWorkerIdFromMetadataFile(String metaFile) {
- Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
- return ObjectReader.getString(map.get("worker-id"), null);
+ return directoryCleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs);
}
private Set<String> getAliveIds(int nowSecs) throws Exception {
@@ -238,8 +194,8 @@ public class LogCleaner implements Runnable, Closeable {
if (logDirs.isEmpty()) {
return new TreeSet<>();
} else {
- Set<String> aliveIds = getAliveIds(nowSecs);
- Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
+ Set<String> aliveIds = workerLogs.getAliveIds(nowSecs);
+ Map<String, File> idToDir = workerLogs.identifyWorkerLogDirs(logDirs);
return idToDir.entrySet().stream()
.filter(entry -> !aliveIds.contains(entry.getKey()))
@@ -249,10 +205,10 @@ public class LogCleaner implements Runnable, Closeable {
}
@VisibleForTesting
- Set<File> selectDirsForCleanup(long nowMillis, String rootDir) {
+ Set<File> selectDirsForCleanup(long nowMillis) {
FileFilter fileFilter = mkFileFilterForLogCleanup(nowMillis);
- return Arrays.stream(new File(rootDir).listFiles())
+ return Arrays.stream(logRootDir.listFiles())
.flatMap(topoDir -> Arrays.stream(topoDir.listFiles(fileFilter)))
.collect(toCollection(TreeSet::new));
}
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
index 1d70b31..6018b79 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
@@ -1,41 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.storm.daemon.logviewer.utils;
+import com.google.common.collect.Lists;
+import org.apache.storm.daemon.supervisor.ClientSupervisorUtils;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Time;
import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+import org.jooq.lambda.tuple.Tuple2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
+import java.util.SortedSet;
import java.util.TreeSet;
import java.util.stream.Stream;
import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.storm.Config.SUPERVISOR_RUN_WORKER_AS_USER;
+import static org.apache.storm.Config.TOPOLOGY_SUBMITTER_USER;
import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
public class WorkerLogs {
+ private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
+
+ public static final String WORKER_YAML = "worker.yaml";
+ private final Map<String, Object> stormConf;
+ private final File logRootDir;
- private WorkerLogs() {
+ public WorkerLogs(Map<String, Object> stormConf, File logRootDir) {
+ this.stormConf = stormConf;
+ this.logRootDir = logRootDir;
}
- /**
- * Return the path of the worker log with the format of topoId/port/worker.log.*
- */
- public static String getTopologyPortWorkerLog(File file) {
- try {
- String[] splitted = file.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR);
- List<String> split = takeLast(Arrays.asList(splitted), 3);
+ public void setLogFilePermission(String fileName) throws IOException {
+ File file = new File(logRootDir, fileName).getCanonicalFile();
+ boolean runAsUser = ObjectReader.getBoolean(stormConf.get(SUPERVISOR_RUN_WORKER_AS_USER), false);
+ File parent = new File(logRootDir, fileName).getParentFile();
+ Optional<File> mdFile = (parent == null) ? Optional.empty() : getMetadataFileForWorkerLogDir(parent);
+ Optional<String> topoOwner = mdFile.isPresent() ?
+ Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().getCanonicalPath())) :
+ Optional.empty();
- return String.join(Utils.FILE_PATH_SEPARATOR, split);
- } catch (IOException e) {
- throw new RuntimeException(e);
+ if (runAsUser && topoOwner.isPresent() && file.exists() && !Files.isReadable(file.toPath())) {
+ LOG.debug("Setting permissions on file {} with topo-owner {}", fileName, topoOwner);
+ ClientSupervisorUtils.processLauncherAndWait(stormConf, topoOwner.get(),
+ Lists.newArrayList("blob", file.getCanonicalPath()), null,
+ "setup group read permissions for file: " + fileName);
}
}
- public static List<File> getAllLogsForRootDir(File logDir) throws IOException {
+ public List<File> getAllLogsForRootDir() throws IOException {
List<File> files = new ArrayList<>();
- Set<File> topoDirFiles = getAllWorkerDirs(logDir);
+ Set<File> topoDirFiles = getAllWorkerDirs();
if (topoDirFiles != null) {
for (File portDir : topoDirFiles) {
files.addAll(DirectoryCleaner.getFilesForDir(portDir));
@@ -45,8 +90,8 @@ public class WorkerLogs {
return files;
}
- public static Set<File> getAllWorkerDirs(File rootDir) {
- File[] rootDirFiles = rootDir.listFiles();
+ public Set<File> getAllWorkerDirs() {
+ File[] rootDirFiles = logRootDir.listFiles();
if (rootDirFiles != null) {
return Arrays.stream(rootDirFiles).flatMap(topoDir -> {
File[] topoFiles = topoDir.listFiles();
@@ -57,4 +102,70 @@ public class WorkerLogs {
return new TreeSet<>();
}
+ /**
+ * Return a sorted set of java.io.Files that were written by workers that are now active.
+ */
+ public SortedSet<String> getAliveWorkerDirs() throws Exception {
+ Set<String> aliveIds = getAliveIds(Time.currentTimeSecs());
+ Set<File> logDirs = getAllWorkerDirs();
+ Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
+
+ return idToDir.entrySet().stream()
+ .filter(entry -> aliveIds.contains(entry.getKey()))
+ .map(Unchecked.function(entry -> entry.getValue().getCanonicalPath()))
+ .collect(toCollection(TreeSet::new));
+ }
+
+ public Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
+ File metaFile = new File(logDir, WORKER_YAML);
+ if (metaFile.exists()) {
+ return Optional.of(metaFile);
+ } else {
+ LOG.warn("Could not find {} to clean up for {}", metaFile.getCanonicalPath(), logDir);
+ return Optional.empty();
+ }
+ }
+
+ public String getWorkerIdFromMetadataFile(String metaFile) {
+ Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
+ return ObjectReader.getString(map.get("worker-id"), null);
+ }
+
+ public String getTopologyOwnerFromMetadataFile(String metaFile) {
+ Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
+ return ObjectReader.getString(map.get(TOPOLOGY_SUBMITTER_USER), null);
+ }
+
+ public Set<String> getAliveIds(int nowSecs) throws Exception {
+ return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
+ .filter(entry -> Objects.nonNull(entry.getValue())
+ && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf))
+ .map(Map.Entry::getKey)
+ .collect(toCollection(TreeSet::new));
+ }
+
+ public Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
+ // we could also make this static, but not to do it due to mock
+ return logDirs.stream().map(Unchecked.function(logDir -> {
+ Optional<File> metaFile = getMetadataFileForWorkerLogDir(logDir);
+
+ return metaFile.map(Unchecked.function(m -> new Tuple2<>(getWorkerIdFromMetadataFile(m.getCanonicalPath()), logDir)))
+ .orElse(new Tuple2<>("", logDir));
+ })).collect(toMap(Tuple2::v1, Tuple2::v2));
+ }
+
+ /**
+ * Return the path of the worker log with the format of topoId/port/worker.log.*
+ */
+ public static String getTopologyPortWorkerLog(File file) {
+ try {
+ String[] splitted = file.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR);
+ List<String> split = takeLast(Arrays.asList(splitted), 3);
+
+ return String.join(Utils.FILE_PATH_SEPARATOR, split);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
index cbf478c..32a723e 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
@@ -28,6 +28,7 @@ import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
import org.apache.storm.security.auth.AuthUtils;
import org.apache.storm.security.auth.IHttpCredentialsPlugin;
import org.apache.storm.utils.ConfigUtils;
@@ -55,11 +56,12 @@ public class LogviewerApplication extends Application {
String daemonLogRoot = logRootDir(ObjectReader.getString(stormConf.get(LOGVIEWER_APPENDER_NAME)));
ResourceAuthorizer resourceAuthorizer = new ResourceAuthorizer(stormConf);
+ WorkerLogs workerLogs = new WorkerLogs(stormConf, new File(logRoot));
- LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, resourceAuthorizer);
+ LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, workerLogs, resourceAuthorizer);
LogviewerProfileHandler profileHandler = new LogviewerProfileHandler(logRoot, resourceAuthorizer);
LogviewerLogDownloadHandler logDownloadHandler = new LogviewerLogDownloadHandler(logRoot, daemonLogRoot,
- resourceAuthorizer);
+ workerLogs, resourceAuthorizer);
LogviewerLogSearchHandler logSearchHandler = new LogviewerLogSearchHandler(stormConf, logRoot, daemonLogRoot,
resourceAuthorizer);
IHttpCredentialsPlugin httpCredsHandler = AuthUtils.GetUiHttpCredentialsPlugin(stormConf);
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
index 4b76abd..2c05972 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
@@ -21,6 +21,7 @@ package org.apache.storm.daemon.logviewer.handler;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
import org.apache.storm.utils.Utils;
import org.assertj.core.util.Lists;
import org.junit.Test;
@@ -31,6 +32,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.attribute.FileAttribute;
import java.util.List;
+import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -56,9 +58,9 @@ public class LogviewerLogPageHandlerTest {
file3.createNewFile();
String origin = "www.origin.server.net";
-
+ Map<String, Object> stormConf = Utils.readStormConfig();
LogviewerLogPageHandler handler = new LogviewerLogPageHandler(rootPath, null,
- new ResourceAuthorizer(Utils.readStormConfig()));
+ new WorkerLogs(stormConf, new File(rootPath)), new ResourceAuthorizer(stormConf));
Response expectedAll = LogviewerResponseBuilder.buildSuccessJsonResponse(
Lists.newArrayList("topoA/port1/worker.log", "topoA/port2/worker.log", "topoB/port1/worker.log"),
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
index 0c7d69c..21f4e9e 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
@@ -80,10 +80,12 @@ public class LogCleanerTest {
conf.put(LOGVIEWER_CLEANUP_AGE_MINS, 60);
conf.put(LOGVIEWER_CLEANUP_INTERVAL_SECS, 300);
- LogCleaner logCleaner = new LogCleaner(conf, mockDirectoryCleaner);
+ WorkerLogs workerLogs = new WorkerLogs(conf, null);
+
+ LogCleaner logCleaner = new LogCleaner(conf, workerLogs, mockDirectoryCleaner, null);
long nowMillis = Time.currentTimeMillis();
- long cutoffMillis = new LogCleaner(conf, mockDirectoryCleaner).cleanupCutoffAgeMillis(nowMillis);
+ long cutoffMillis = logCleaner.cleanupCutoffAgeMillis(nowMillis);
long oldMtimeMillis = cutoffMillis - 500;
long newMtimeMillis = cutoffMillis + 500;
@@ -125,9 +127,6 @@ public class LogCleanerTest {
when(mockDirectoryCleaner.deleteOldestWhileTooLarge(anyListOf(File.class), anyLong(), anyBoolean(), anySetOf(String.class)))
.thenCallRealMethod();
- Map<String, Object> conf = Utils.readStormConfig();
- LogCleaner logCleaner = new LogCleaner(conf, mockDirectoryCleaner);
-
long nowMillis = Time.currentTimeMillis();
List<File> files1 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("A" + idx)
@@ -157,7 +156,11 @@ public class LogCleanerTest {
File rootDir = new MockDirectoryBuilder().setDirName("/workers-artifacts")
.setFiles(rootFiles).build();
- List<Integer> deletedFiles = logCleaner.perWorkerDirCleanup(rootDir, 1200, mockDirectoryCleaner);
+ Map<String, Object> conf = Utils.readStormConfig();
+ WorkerLogs workerLogs = new WorkerLogs(conf, rootDir);
+ LogCleaner logCleaner = new LogCleaner(conf, workerLogs, mockDirectoryCleaner, rootDir);
+
+ List<Integer> deletedFiles = logCleaner.perWorkerDirCleanup(1200);
assertEquals(Integer.valueOf(4), deletedFiles.get(0));
assertEquals(Integer.valueOf(4), deletedFiles.get(1));
assertEquals(Integer.valueOf(4), deletedFiles.get(deletedFiles.size() - 1));
@@ -182,15 +185,6 @@ public class LogCleanerTest {
when(mockDirectoryCleaner.deleteOldestWhileTooLarge(anyListOf(File.class), anyLong(), anyBoolean(), anySetOf(String.class)))
.thenCallRealMethod();
- Map<String, Object> conf = Utils.readStormConfig();
-
- LogCleaner logCleaner = new LogCleaner(conf, mockDirectoryCleaner) {
- @Override
- SortedSet<String> getAliveWorkerDirs(File rootDir) throws Exception {
- return new TreeSet<>(Collections.singletonList("/workers-artifacts/topo1/port1"));
- }
- };
-
long nowMillis = Time.currentTimeMillis();
List<File> files1 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("A" + idx + ".log")
@@ -222,45 +216,19 @@ public class LogCleanerTest {
File rootDir = new MockDirectoryBuilder().setDirName("/workers-artifacts")
.setFiles(rootFiles).build();
- int deletedFiles = logCleaner.globalLogCleanup(rootDir, 2400, mockDirectoryCleaner);
- assertEquals(18, deletedFiles);
- } finally {
- Utils.setInstance(prevUtils);
- }
- }
-
- /**
- * Build up workerid-workerlogdir map for the old workers' dirs.
- */
- @Test
- public void testIdentifyWorkerLogDirs() throws Exception {
- File port1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port1").build();
- File mockMetaFile = new MockFileBuilder().setFileName("worker.yaml").build();
-
- String expId = "id12345";
- Map<String, File> expected = Collections.singletonMap(expId, port1Dir);
-
- try {
- SupervisorUtils mockedSupervisorUtils = mock(SupervisorUtils.class);
- SupervisorUtils.setInstance(mockedSupervisorUtils);
-
- Map<String, Object> stormConf = Utils.readStormConfig();
- LogCleaner logCleaner = new LogCleaner(stormConf, new DirectoryCleaner()) {
- @Override
- Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
- return Optional.of(mockMetaFile);
- }
-
+ Map<String, Object> conf = Utils.readStormConfig();
+ WorkerLogs stubbedWorkerLogs = new WorkerLogs(conf, rootDir) {
@Override
- String getWorkerIdFromMetadataFile(String metaFile) {
- return expId;
+ public SortedSet<String> getAliveWorkerDirs() throws Exception {
+ return new TreeSet<>(Collections.singletonList("/workers-artifacts/topo1/port1"));
}
};
- when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMapOf(String.class, Object.class))).thenReturn(null);
- assertEquals(expected, logCleaner.identifyWorkerLogDirs(Collections.singleton(port1Dir)));
+ LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, mockDirectoryCleaner, rootDir);
+ int deletedFiles = logCleaner.globalLogCleanup(2400);
+ assertEquals(18, deletedFiles);
} finally {
- SupervisorUtils.resetInstance();
+ Utils.setInstance(prevUtils);
}
}
@@ -286,9 +254,10 @@ public class LogCleanerTest {
SupervisorUtils mockedSupervisorUtils = mock(SupervisorUtils.class);
SupervisorUtils.setInstance(mockedSupervisorUtils);
- LogCleaner logCleaner = new LogCleaner(stormConf, new DirectoryCleaner()) {
+ Map<String, Object> conf = Utils.readStormConfig();
+ WorkerLogs stubbedWorkerLogs = new WorkerLogs(conf, null) {
@Override
- Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
+ public Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
Map<String, File> ret = new HashMap<>();
ret.put("42", unexpectedDir1);
ret.put("007", expectedDir2);
@@ -299,6 +268,8 @@ public class LogCleanerTest {
}
};
+ LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, new DirectoryCleaner(), null);
+
when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMapOf(String.class, Object.class))).thenReturn(idToHb);
assertEquals(Sets.newSet(expectedDir2, expectedDir3), logCleaner.getDeadWorkerDirs(nowSecs, logDirs));
} finally {
@@ -326,9 +297,13 @@ public class LogCleanerTest {
return null;
}).when(mockUtils).forceDelete(anyString());
- LogCleaner logCleaner = new LogCleaner(Utils.readStormConfig(), new DirectoryCleaner()) {
+
+ Map<String, Object> conf = Utils.readStormConfig();
+ WorkerLogs stubbedWorkerLogs = new WorkerLogs(conf, null);
+
+ LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, new DirectoryCleaner(), null) {
@Override
- Set<File> selectDirsForCleanup(long nowMillis, String rootDir) {
+ Set<File> selectDirsForCleanup(long nowMillis) {
return Collections.emptySet();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/bb5c247f/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java
new file mode 100644
index 0000000..b8b5d8c
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.utils;
+
+import org.apache.storm.daemon.logviewer.testsupport.MockDirectoryBuilder;
+import org.apache.storm.daemon.logviewer.testsupport.MockFileBuilder;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.utils.Utils;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyMapOf;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class WorkerLogsTest {
+
+ /**
+ * Build up workerid-workerlogdir map for the old workers' dirs.
+ */
+ @Test
+ public void testIdentifyWorkerLogDirs() throws Exception {
+ File port1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port1").build();
+ File mockMetaFile = new MockFileBuilder().setFileName("worker.yaml").build();
+
+ String expId = "id12345";
+ Map<String, File> expected = Collections.singletonMap(expId, port1Dir);
+
+ try {
+ SupervisorUtils mockedSupervisorUtils = mock(SupervisorUtils.class);
+ SupervisorUtils.setInstance(mockedSupervisorUtils);
+
+ Map<String, Object> stormConf = Utils.readStormConfig();
+ WorkerLogs workerLogs = new WorkerLogs(stormConf, port1Dir) {
+ @Override
+ public Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
+ return Optional.of(mockMetaFile);
+ }
+
+ @Override
+ public String getWorkerIdFromMetadataFile(String metaFile) {
+ return expId;
+ }
+ };
+
+ when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMapOf(String.class, Object.class))).thenReturn(null);
+ assertEquals(expected, workerLogs.identifyWorkerLogDirs(Collections.singleton(port1Dir)));
+ } finally {
+ SupervisorUtils.resetInstance();
+ }
+ }
+
+}
[07/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/test/clj/org/apache/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
deleted file mode 100644
index ae9b651..0000000
--- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
+++ /dev/null
@@ -1,824 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.logviewer-test
- (:use [org.apache.storm daemon-config config util])
- (:require [org.apache.storm.daemon [logviewer :as logviewer]])
- (:require [conjure.core])
- (:use [clojure test])
- (:use [conjure core])
- (:use [org.apache.storm.ui helpers])
- (:import [org.apache.storm.daemon DirectoryCleaner]
- [org.apache.storm.utils Time Utils]
- [org.apache.storm.utils.staticmocking UtilsInstaller]
- [org.apache.storm.daemon.supervisor SupervisorUtils]
- [org.apache.storm.testing.staticmocking MockedSupervisorUtils]
- [org.apache.storm.generated LSWorkerHeartbeat])
- (:import [java.nio.file Files Path DirectoryStream])
- (:import [java.nio.file Files])
- (:import [java.nio.file.attribute FileAttribute])
- (:import [java.io File])
- (:import [java.util ArrayList])
- (:import [org.mockito Mockito]))
-
-(defn mk-mock-Path [file]
- (let [mockPath (Mockito/mock java.nio.file.Path)]
- (. (Mockito/when (.toFile mockPath)) thenReturn file)
- mockPath))
-
-(defn mk-DirectoryStream [^ArrayList list-of-paths]
- (reify DirectoryStream
- (close [this])
- (iterator [this]
- (.iterator list-of-paths))))
-
-(defmulti mk-mock-File #(:type %))
-
-(defmethod mk-mock-File :file [{file-name :name mtime :mtime length :length
- :or {file-name "afile"
- mtime 1
- length (* 10 (* 1024 (* 1024 1024))) }}] ; Length 10 GB
- (let [mockFile (Mockito/mock java.io.File)]
- (. (Mockito/when (.getName mockFile)) thenReturn file-name)
- (. (Mockito/when (.lastModified mockFile)) thenReturn mtime)
- (. (Mockito/when (.isFile mockFile)) thenReturn true)
- (. (Mockito/when (.getCanonicalPath mockFile))
- thenReturn (str "/mock/canonical/path/to/" file-name))
- (. (Mockito/when (.length mockFile)) thenReturn length)
- mockFile))
-
-(defmethod mk-mock-File :directory [{dir-name :name mtime :mtime files :files
- :or {dir-name "adir" mtime 1 files []}}]
- (let [mockDir (Mockito/mock java.io.File)]
- (. (Mockito/when (.getName mockDir)) thenReturn dir-name)
- (. (Mockito/when (.lastModified mockDir)) thenReturn mtime)
- (. (Mockito/when (.isFile mockDir)) thenReturn false)
- (. (Mockito/when (.listFiles mockDir)) thenReturn (into-array File files))
- (. (Mockito/when (.getCanonicalPath mockDir)) thenReturn dir-name)
- mockDir))
-
-(deftest test-get-size-for-logdir
- (testing "get the file sizes of a worker log directory"
- (stubbing [logviewer/get-stream-for-dir (fn [x] (map #(mk-mock-Path %) (.listFiles x)))]
- (let [now-millis (Time/currentTimeMillis)
- files1 (into-array File (map #(mk-mock-File {:name (str %)
- :type :file
- :mtime (- now-millis (* 100 %))
- :length 200})
- (range 1 6))) ;; 5 files
- port1-dir (mk-mock-File {:name "/workers-artifacts/topo1/port1"
- :type :directory
- :files files1})]
- (is (= (logviewer/get-size-for-logdir port1-dir) 1000))))))
-
-(deftest test-mk-FileFilter-for-log-cleanup
- (testing "log file filter selects the correct worker-log dirs for purge"
- (stubbing [logviewer/get-stream-for-dir (fn [x] (map #(mk-mock-Path %) (.listFiles x)))]
- (let [now-millis (Time/currentTimeMillis)
- conf {LOGVIEWER-CLEANUP-AGE-MINS 60
- LOGVIEWER-CLEANUP-INTERVAL-SECS 300}
- cutoff-millis (logviewer/cleanup-cutoff-age-millis conf now-millis)
- old-mtime-millis (- cutoff-millis 500)
- new-mtime-millis (+ cutoff-millis 500)
- matching-files (map #(mk-mock-File %)
- [{:name "3031"
- :type :directory
- :mtime old-mtime-millis}
- {:name "3032"
- :type :directory
- :mtime old-mtime-millis}
- {:name "7077"
- :type :directory
- :mtime old-mtime-millis}])
- excluded-files (map #(mk-mock-File %)
- [{:name "oldlog-1-2-worker-.log"
- :type :file
- :mtime old-mtime-millis}
- {:name "newlog-1-2-worker.log"
- :type :file
- :mtime new-mtime-millis}
- {:name "some-old-file.txt"
- :type :file
- :mtime old-mtime-millis}
- {:name "olddir-1-2-worker.log"
- :type :directory
- :mtime new-mtime-millis}
- {:name "metadata"
- :type :directory
- :mtime new-mtime-millis}
- {:name "newdir"
- :type :directory
- :mtime new-mtime-millis}
- ])
- file-filter (logviewer/mk-FileFilter-for-log-cleanup conf now-millis)]
- (is (every? #(.accept file-filter %) matching-files))
- (is (not-any? #(.accept file-filter %) excluded-files))
- ))))
-
-(deftest test-per-workerdir-cleanup!
- (testing "cleaner deletes oldest files in each worker dir if files are larger than per-dir quota."
- (with-open [_ (UtilsInstaller. (proxy [Utils] []
- (forceDeleteImpl [path])))]
- (let [cleaner (proxy [org.apache.storm.daemon.DirectoryCleaner] []
- (getStreamForDirectory
- ([^File dir]
- (mk-DirectoryStream
- (ArrayList.
- (map #(mk-mock-Path %) (.listFiles dir)))))))
- now-millis (Time/currentTimeMillis)
- files1 (into-array File (map #(mk-mock-File {:name (str "A" %)
- :type :file
- :mtime (+ now-millis (* 100 %))
- :length 200 })
- (range 0 10)))
- files2 (into-array File (map #(mk-mock-File {:name (str "B" %)
- :type :file
- :mtime (+ now-millis (* 100 %))
- :length 200 })
- (range 0 10)))
- files3 (into-array File (map #(mk-mock-File {:name (str "C" %)
- :type :file
- :mtime (+ now-millis (* 100 %))
- :length 200 })
- (range 0 10)))
- port1-dir (mk-mock-File {:name "/workers-artifacts/topo1/port1"
- :type :directory
- :files files1})
- port2-dir (mk-mock-File {:name "/workers-artifacts/topo1/port2"
- :type :directory
- :files files2})
- port3-dir (mk-mock-File {:name "/workers-artifacts/topo2/port3"
- :type :directory
- :files files3})
- topo1-files (into-array File [port1-dir port2-dir])
- topo2-files (into-array File [port3-dir])
- topo1-dir (mk-mock-File {:name "/workers-artifacts/topo1"
- :type :directory
- :files topo1-files})
- topo2-dir (mk-mock-File {:name "/workers-artifacts/topo2"
- :type :directory
- :files topo2-files})
- root-files (into-array File [topo1-dir topo2-dir])
- root-dir (mk-mock-File {:name "/workers-artifacts"
- :type :directory
- :files root-files})
- deletedFiles (logviewer/per-workerdir-cleanup! root-dir 1200 cleaner)]
- (is (= (first deletedFiles) 4))
- (is (= (second deletedFiles) 4))
- (is (= (last deletedFiles) 4))))))
-
-(deftest test-global-log-cleanup!
- (testing "cleaner deletes oldest when files' sizes are larger than the global quota."
- (stubbing [logviewer/get-alive-worker-dirs ["/workers-artifacts/topo1/port1"]]
- (with-open [_ (UtilsInstaller. (proxy [Utils] []
- (forceDeleteImpl [path])))]
- (let [cleaner (proxy [org.apache.storm.daemon.DirectoryCleaner] []
- (getStreamForDirectory
- ([^File dir]
- (mk-DirectoryStream
- (ArrayList.
- (map #(mk-mock-Path %) (.listFiles dir)))))))
- now-millis (Time/currentTimeMillis)
- files1 (into-array File (map #(mk-mock-File {:name (str "A" % ".log")
- :type :file
- :mtime (+ now-millis (* 100 %))
- :length 200 })
- (range 0 10)))
- files2 (into-array File (map #(mk-mock-File {:name (str "B" %)
- :type :file
- :mtime (+ now-millis (* 100 %))
- :length 200 })
- (range 0 10)))
- files3 (into-array File (map #(mk-mock-File {:name (str "C" %)
- :type :file
- :mtime (+ now-millis (* 100 %))
- :length 200 })
- (range 0 10)))
- port1-dir (mk-mock-File {:name "/workers-artifacts/topo1/port1"
- :type :directory
- :files files1}) ;; note that port1-dir is active worker containing active logs
- port2-dir (mk-mock-File {:name "/workers-artifacts/topo1/port2"
- :type :directory
- :files files2})
- port3-dir (mk-mock-File {:name "/workers-artifacts/topo2/port3"
- :type :directory
- :files files3})
- topo1-files (into-array File [port1-dir port2-dir])
- topo2-files (into-array File [port3-dir])
- topo1-dir (mk-mock-File {:name "/workers-artifacts/topo1"
- :type :directory
- :files topo1-files})
- topo2-dir (mk-mock-File {:name "/workers-artifacts/topo2"
- :type :directory
- :files topo2-files})
- root-files (into-array File [topo1-dir topo2-dir])
- root-dir (mk-mock-File {:name "/workers-artifacts"
- :type :directory
- :files root-files})
- deletedFiles (logviewer/global-log-cleanup! root-dir 2400 cleaner)]
- (is (= deletedFiles 18)))))))
-
-(deftest test-identify-worker-log-dirs
- (testing "Build up workerid-workerlogdir map for the old workers' dirs"
- (let [port1-dir (mk-mock-File {:name "/workers-artifacts/topo1/port1"
- :type :directory})
- mock-metaFile (mk-mock-File {:name "worker.yaml"
- :type :file})
- exp-id "id12345"
- expected {exp-id port1-dir}
- supervisor-util (Mockito/mock SupervisorUtils)]
- (with-open [_ (MockedSupervisorUtils. supervisor-util)]
- (stubbing [logviewer/get-metadata-file-for-wroker-logdir mock-metaFile
- logviewer/get-worker-id-from-metadata-file exp-id]
- (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn nil))
- (is (= expected (logviewer/identify-worker-log-dirs [port1-dir]))))))))
-
-(deftest test-get-dead-worker-dirs
- (testing "return directories for workers that are not alive"
- (let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
- hb (let [lwb (LSWorkerHeartbeat.)]
- (.set_time_secs lwb (int 1)) lwb)
- id->hb {"42" hb}
- now-secs 2
- unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory})
- expected-dir2 (mk-mock-File {:name "dir2" :type :directory})
- expected-dir3 (mk-mock-File {:name "dir3" :type :directory})
- log-dirs #{unexpected-dir1 expected-dir2 expected-dir3}
- supervisor-util (Mockito/mock SupervisorUtils)]
- (with-open [_ (MockedSupervisorUtils. supervisor-util)]
- (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir1,
- "007" expected-dir2,
- "" expected-dir3}] ;; this tests a directory with no yaml file thus no worker id
- (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn id->hb))
- (is (= #{expected-dir2 expected-dir3}
- (logviewer/get-dead-worker-dirs conf now-secs log-dirs))))))))
-
-(deftest test-cleanup-fn
- (testing "cleanup function forceDeletes files of dead workers"
- (let [mockfile1 (mk-mock-File {:name "delete-me1" :type :file})
- mockfile2 (mk-mock-File {:name "delete-me2" :type :file})
- forceDelete-args (atom [])
- utils-proxy (proxy [Utils] []
- (forceDeleteImpl [path]
- (swap! forceDelete-args conj path)))]
- (with-open [_ (UtilsInstaller. utils-proxy)]
- (stubbing [logviewer/select-dirs-for-cleanup nil
- logviewer/get-dead-worker-dirs (sorted-set mockfile1 mockfile2)
- logviewer/cleanup-empty-topodir! nil]
- (logviewer/cleanup-fn! "/bogus/path")
- (is (= 2 (count @forceDelete-args)))
- (is (= (.getCanonicalPath mockfile1) (get @forceDelete-args 0)))
- (is (= (.getCanonicalPath mockfile2) (get @forceDelete-args 1))))))))
-
-(deftest test-authorized-log-user
- (testing "allow cluster admin"
- (let [conf {NIMBUS-ADMINS ["alice"]}]
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]
- logviewer/user-groups []]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf))
- (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
- (verify-first-call-args-for logviewer/user-groups "alice"))))
-
- (testing "ignore any cluster-set topology.users topology.groups"
- (let [conf {TOPOLOGY-USERS ["alice"]
- TOPOLOGY-GROUPS ["alice-group"]}]
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]
- logviewer/user-groups ["alice-group"]]
- (is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" conf)))
- (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
- (verify-first-call-args-for logviewer/user-groups "alice"))))
-
- (testing "allow cluster logs user"
- (let [conf {LOGS-USERS ["alice"]}]
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]
- logviewer/user-groups []]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" conf))
- (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
- (verify-first-call-args-for logviewer/user-groups "alice"))))
-
- (testing "allow whitelisted topology user"
- (stubbing [logviewer/get-log-user-group-whitelist [["alice"] []]
- logviewer/user-groups []]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))
- (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
- (verify-first-call-args-for logviewer/user-groups "alice")))
-
- (testing "allow whitelisted topology group"
- (stubbing [logviewer/get-log-user-group-whitelist [[] ["alice-group"]]
- logviewer/user-groups ["alice-group"]]
- (is (logviewer/authorized-log-user? "alice" "non-blank-fname" {}))
- (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
- (verify-first-call-args-for logviewer/user-groups "alice")))
-
- (testing "disallow user not in nimbus admin, topo user, logs user, or whitelist"
- (stubbing [logviewer/get-log-user-group-whitelist [[] []]
- logviewer/user-groups []]
- (is (not (logviewer/authorized-log-user? "alice" "non-blank-fname" {})))
- (verify-first-call-args-for logviewer/get-log-user-group-whitelist "non-blank-fname")
- (verify-first-call-args-for logviewer/user-groups "alice"))))
-
-(deftest test-list-log-files
- (testing "list-log-files filter selects the correct log files to return"
- (let [attrs (make-array FileAttribute 0)
- root-path (.getCanonicalPath (.toFile (Files/createTempDirectory "workers-artifacts" attrs)))
- file1 (clojure.java.io/file root-path "topoA" "port1" "worker.log")
- file2 (clojure.java.io/file root-path "topoA" "port2" "worker.log")
- file3 (clojure.java.io/file root-path "topoB" "port1" "worker.log")
- _ (clojure.java.io/make-parents file1)
- _ (clojure.java.io/make-parents file2)
- _ (clojure.java.io/make-parents file3)
- _ (.createNewFile file1)
- _ (.createNewFile file2)
- _ (.createNewFile file3)
- origin "www.origin.server.net"
- expected-all (json-response '("topoA/port1/worker.log" "topoA/port2/worker.log"
- "topoB/port1/worker.log")
- nil
- :headers {"Access-Control-Allow-Origin" origin
- "Access-Control-Allow-Credentials" "true"})
- expected-filter-port (json-response '("topoA/port1/worker.log" "topoB/port1/worker.log")
- nil
- :headers {"Access-Control-Allow-Origin" origin
- "Access-Control-Allow-Credentials" "true"})
- expected-filter-topoId (json-response '("topoB/port1/worker.log")
- nil
- :headers {"Access-Control-Allow-Origin" origin
- "Access-Control-Allow-Credentials" "true"})
- returned-all (logviewer/list-log-files "user" nil nil root-path nil origin)
- returned-filter-port (logviewer/list-log-files "user" nil "port1" root-path nil origin)
- returned-filter-topoId (logviewer/list-log-files "user" "topoB" nil root-path nil origin)]
- (Utils/forceDelete root-path)
- (is (= expected-all returned-all))
- (is (= expected-filter-port returned-filter-port))
- (is (= expected-filter-topoId returned-filter-topoId)))))
-
-(deftest test-search-via-rest-api
- (testing "Throws if bogus file is given"
- (thrown-cause? java.lang.RuntimeException
- (logviewer/substring-search nil "a string")))
-
- (let [pattern "needle"
- expected-host "dev.null.invalid"
- expected-port 8888
- ;; When we click a link to the logviewer, we expect the match line to
- ;; be somewhere near the middle of the page. So we subtract half of
- ;; the default page length from the offset at which we found the
- ;; match.
- exp-offset-fn #(- (/ logviewer/default-bytes-per-page 2) %)]
-
- (stubbing [logviewer/logviewer-port expected-port]
- (with-open [_ (UtilsInstaller. (proxy [Utils] []
- (hostnameImpl [] expected-host)))]
- (testing "Logviewer link centers the match in the page"
- (let [expected-fname "foobar.log"]
- (is (= (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file="
- expected-fname
- "&start=1947&length="
- logviewer/default-bytes-per-page)
- (logviewer/url-to-match-centered-in-log-page (byte-array 42)
- expected-fname
- 27526
- 8888)))))
-
- (testing "Logviewer link centers the match in the page (daemon)"
- (let [expected-fname "foobar.log"]
- (is (= (str "http://"
- expected-host
- ":"
- expected-port
- "/daemonlog?file="
- expected-fname
- "&start=1947&length="
- logviewer/default-bytes-per-page)
- (logviewer/url-to-match-centered-in-log-page-daemon-file (byte-array 42)
- expected-fname
- 27526
- 8888)))))
-
- (let [file (->> "logviewer-search-context-tests.log.test"
- (clojure.java.io/file "src" "dev"))]
- (testing "returns correct before/after context"
- (is (= {"isDaemon" "no"
- "searchString" pattern
- "startByteOffset" 0
- "matches" [{"byteOffset" 0
- "beforeString" ""
- "afterString" " needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle "
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 7
- "beforeString" "needle "
- "afterString" "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle needle\n"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 127
- "beforeString" "needle needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
- "afterString" " needle\n"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 134
- "beforeString" " needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle "
- "afterString" "\n"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- ]}
- (logviewer/substring-search file pattern)))))
-
- (let [file (clojure.java.io/file "src" "dev" "small-worker.log.test")]
- (testing "a really small log file"
- (is (= {"isDaemon" "no"
- "searchString" pattern
- "startByteOffset" 0
- "matches" [{"byteOffset" 7
- "beforeString" "000000 "
- "afterString" " 000000\n"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}]}
- (logviewer/substring-search file pattern)))))
-
- (let [file (clojure.java.io/file "src" "dev" "small-worker.log.test")]
- (testing "a really small log file (daemon)"
- (is (= {"isDaemon" "yes"
- "searchString" pattern
- "startByteOffset" 0
- "matches" [{"byteOffset" 7
- "beforeString" "000000 "
- "afterString" " 000000\n"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/daemonlog?file="
- (.getName file)
- "&start=0&length=51200")}]}
- (logviewer/substring-search file pattern :is-daemon true)))))
-
- (let [file (clojure.java.io/file "src" "dev" "test-3072.log.test")]
- (testing "no offset returned when file ends on buffer offset"
- (let [expected
- {"isDaemon" "no"
- "searchString" pattern
- "startByteOffset" 0
- "matches" [{"byteOffset" 3066
- "beforeString" (->>
- (repeat 128 '.)
- clojure.string/join)
- "afterString" ""
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}]}]
- (is (= expected
- (logviewer/substring-search file pattern)))
- (is (= expected
- (logviewer/substring-search file pattern :num-matches 1))))))
-
- (let [file (clojure.java.io/file "src" "dev" "test-worker.log.test")]
-
- (testing "next byte offsets are correct for each match"
- (doseq [[num-matches-sought
- num-matches-found
- expected-next-byte-offset] [[1 1 11]
- [2 2 2042]
- [3 3 2052]
- [4 4 3078]
- [5 5 3196]
- [6 6 3202]
- [7 7 6252]
- [8 8 6321]
- [9 9 6397]
- [10 10 6476]
- [11 11 6554]
- [12 12 nil]
- [13 12 nil]]]
- (let [result
- (logviewer/substring-search file
- pattern
- :num-matches num-matches-sought)]
- (is (= expected-next-byte-offset
- (get result "nextByteOffset")))
- (is (= num-matches-found (count (get result "matches")))))))
-
- (is
- (= {"isDaemon" "no"
- "nextByteOffset" 6252
- "searchString" pattern
- "startByteOffset" 0
- "matches" [
- {"byteOffset" 5
- "beforeString" "Test "
- "afterString" " is near the beginning of the file.\nThis file assumes a buffer size of 2048 bytes, a max search string size of 1024 bytes, and a"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 2036
- "beforeString" "ng 146\npadding 147\npadding 148\npadding 149\npadding 150\npadding 151\npadding 152\npadding 153\nNear the end of a 1024 byte block, a "
- "afterString" ".\nA needle that straddles a 1024 byte boundary should also be detected.\n\npadding 157\npadding 158\npadding 159\npadding 160\npadding"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 2046
- "beforeString" "ding 147\npadding 148\npadding 149\npadding 150\npadding 151\npadding 152\npadding 153\nNear the end of a 1024 byte block, a needle.\nA "
- "afterString" " that straddles a 1024 byte boundary should also be detected.\n\npadding 157\npadding 158\npadding 159\npadding 160\npadding 161\npaddi"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 3072
- "beforeString" "adding 226\npadding 227\npadding 228\npadding 229\npadding 230\npadding 231\npadding 232\npadding 233\npadding 234\npadding 235\n\n\nHere a "
- "afterString" " occurs just after a 1024 byte boundary. It should have the correct context.\n\nText with two adjoining matches: needleneedle\n\npa"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 3190
- "beforeString" "\n\n\nHere a needle occurs just after a 1024 byte boundary. It should have the correct context.\n\nText with two adjoining matches: "
- "afterString" "needle\n\npadding 243\npadding 244\npadding 245\npadding 246\npadding 247\npadding 248\npadding 249\npadding 250\npadding 251\npadding 252\n"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 3196
- "beforeString" "e a needle occurs just after a 1024 byte boundary. It should have the correct context.\n\nText with two adjoining matches: needle"
- "afterString" "\n\npadding 243\npadding 244\npadding 245\npadding 246\npadding 247\npadding 248\npadding 249\npadding 250\npadding 251\npadding 252\npaddin"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 6246
- "beforeString" "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n\nHere are four non-ascii 1-byte UTF-8 characters: αβγδε\n\n"
- "afterString" "\n\nHere are four printable 2-byte UTF-8 characters: ¡¢£¤¥\n\nneedle\n\n\n\nHere are four printable 3-byte UTF-8 characters: ऄअ"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- ]}
- (logviewer/substring-search file pattern :num-matches 7)))
-
- (testing "Correct match offset is returned when skipping bytes"
- (let [start-byte-offset 3197]
- (is (= {"isDaemon" "no"
- "nextByteOffset" 6252
- "searchString" pattern
- "startByteOffset" start-byte-offset
- "matches" [{"byteOffset" 6246
- "beforeString" "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n\nHere are four non-ascii 1-byte UTF-8 characters: αβγδε\n\n"
- "afterString" "\n\nHere are four printable 2-byte UTF-8 characters: ¡¢£¤¥\n\nneedle\n\n\n\nHere are four printable 3-byte UTF-8 characters: ऄअ"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}]}
- (logviewer/substring-search file
- pattern
- :num-matches 1
- :start-byte-offset start-byte-offset)))))
-
- (let [pattern (clojure.string/join (repeat 1024 'X))]
- (is
- (= {"isDaemon" "no"
- "nextByteOffset" 6183
- "searchString" pattern
- "startByteOffset" 0
- "matches" [
- {"byteOffset" 4075
- "beforeString" "\n\nThe following match of 1024 bytes completely fills half the byte buffer. It is a search substring of the maximum size......\n\n"
- "afterString" "\nThe following max-size match straddles a 1024 byte buffer.\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- {"byteOffset" 5159
- "beforeString" "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\nThe following max-size match straddles a 1024 byte buffer.\n"
- "afterString" "\n\nHere are four non-ascii 1-byte UTF-8 characters: αβγδε\n\nneedle\n\nHere are four printable 2-byte UTF-8 characters: ¡¢£¤"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- ]}
- (logviewer/substring-search file pattern :num-matches 2))))
-
- (let [pattern "𐄀𐄁𐄂"]
- (is
- (= {"isDaemon" "no"
- "nextByteOffset" 7176
- "searchString" pattern
- "startByteOffset" 0
- "matches" [
- {"byteOffset" 7164
- "beforeString" "padding 372\npadding 373\npadding 374\npadding 375\n\nThe following tests multibyte UTF-8 Characters straddling the byte boundary: "
- "afterString" "\n\nneedle"
- "matchString" pattern
- "logviewerURL" (str "http://"
- expected-host
- ":"
- expected-port
- "/log?file=src%2Fdev%2F"
- (.getName file)
- "&start=0&length=51200")}
- ]}
- (logviewer/substring-search file pattern :num-matches 1))))
-
- (testing "Returns 0 matches for unseen pattern"
- (let [pattern "Not There"]
- (is (= {"isDaemon" "no"
- "searchString" pattern
- "startByteOffset" 0
- "matches" []}
- (logviewer/substring-search file
- pattern
- :num-matches nil
- :start-byte-offset nil))))))))))
-
-(deftest test-find-n-matches
- (testing "find-n-matches looks through logs properly"
- (let [files [(clojure.java.io/file "src" "dev" "logviewer-search-context-tests.log.test")
- (clojure.java.io/file "src" "dev" "logviewer-search-context-tests.log.gz")]
- matches1 ((logviewer/find-n-matches files 20 0 0 "needle") "matches")
- matches2 ((logviewer/find-n-matches files 20 0 126 "needle") "matches")
- matches3 ((logviewer/find-n-matches files 20 1 0 "needle") "matches")]
-
- (is (= 2 (count matches1)))
- (is (= 4 (count ((first matches1) "matches"))))
- (is (= 4 (count ((second matches1) "matches"))))
- (is (= ((first matches1) "fileName") "src/dev/logviewer-search-context-tests.log.test"))
- (is (= ((second matches1) "fileName") "src/dev/logviewer-search-context-tests.log.gz"))
-
- (is (= 2 (count ((first matches2) "matches"))))
- (is (= 4 (count ((second matches2) "matches"))))
-
- (is (= 1 (count matches3)))
- (is (= 4 (count ((first matches3) "matches")))))))
-
-(deftest test-deep-search-logs-for-topology
- (let [files [(clojure.java.io/file "src" "dev" "logviewer-search-context-tests.log.test")
- (clojure.java.io/file "src" "dev" "logviewer-search-context-tests.log.gz")]
- attrs (make-array FileAttribute 0)
- topo-path (.getCanonicalPath (.toFile (Files/createTempDirectory "topoA" attrs)))
- _ (.createNewFile (clojure.java.io/file topo-path "6400"))
- _ (.createNewFile (clojure.java.io/file topo-path "6500"))
- _ (.createNewFile (clojure.java.io/file topo-path "6600"))
- _ (.createNewFile (clojure.java.io/file topo-path "6700"))]
- (stubbing
- [logviewer/logs-for-port files
- logviewer/find-n-matches nil]
- (testing "deep-search-logs-for-topology all-ports search-archived = true"
- (instrumenting
- [logviewer/find-n-matches
- logviewer/logs-for-port]
- (logviewer/deep-search-logs-for-topology "" nil topo-path "search" "20" "*" "20" "199" true nil nil)
- (verify-call-times-for logviewer/find-n-matches 4)
- (verify-call-times-for logviewer/logs-for-port 4)
- ; File offset and byte offset should always be zero when searching multiple workers (multiple ports).
- (verify-nth-call-args-for 1 logviewer/find-n-matches files 20 0 0 "search")
- (verify-nth-call-args-for 2 logviewer/find-n-matches files 20 0 0 "search")
- (verify-nth-call-args-for 3 logviewer/find-n-matches files 20 0 0 "search")
- (verify-nth-call-args-for 4 logviewer/find-n-matches files 20 0 0 "search")))
- (testing "deep-search-logs-for-topology all-ports search-archived = false"
- (instrumenting
- [logviewer/find-n-matches
- logviewer/logs-for-port]
- (logviewer/deep-search-logs-for-topology "" nil topo-path "search" "20" nil "20" "199" nil nil nil)
- (verify-call-times-for logviewer/find-n-matches 4)
- (verify-call-times-for logviewer/logs-for-port 4)
- ; File offset and byte offset should always be zero when searching multiple workers (multiple ports).
- (verify-nth-call-args-for 1 logviewer/find-n-matches [(first files)] 20 0 0 "search")
- (verify-nth-call-args-for 2 logviewer/find-n-matches [(first files)] 20 0 0 "search")
- (verify-nth-call-args-for 3 logviewer/find-n-matches [(first files)] 20 0 0 "search")
- (verify-nth-call-args-for 4 logviewer/find-n-matches [(first files)] 20 0 0 "search")))
- (testing "deep-search-logs-for-topology one-port search-archived = true, no file-offset"
- (instrumenting
- [logviewer/find-n-matches
- logviewer/logs-for-port]
- (logviewer/deep-search-logs-for-topology "" nil topo-path "search" "20" "6700" "0" "0" true nil nil)
- (verify-call-times-for logviewer/find-n-matches 1)
- (verify-call-times-for logviewer/logs-for-port 2)
- (verify-nth-call-args-for 1 logviewer/find-n-matches files 20 0 0 "search")))
- (testing "deep-search-logs-for-topology one-port search-archived = true, file-offset = 1"
- (instrumenting
- [logviewer/find-n-matches
- logviewer/logs-for-port]
- (logviewer/deep-search-logs-for-topology "" nil topo-path "search" "20" "6700" "1" "0" true nil nil)
- (verify-call-times-for logviewer/find-n-matches 1)
- (verify-call-times-for logviewer/logs-for-port 2)
- (verify-nth-call-args-for 1 logviewer/find-n-matches files 20 1 0 "search")))
- (testing "deep-search-logs-for-topology one-port search-archived = false, file-offset = 1"
- (instrumenting
- [logviewer/find-n-matches
- logviewer/logs-for-port]
- (logviewer/deep-search-logs-for-topology "" nil topo-path "search" "20" "6700" "1" "0" nil nil nil)
- (verify-call-times-for logviewer/find-n-matches 1)
- (verify-call-times-for logviewer/logs-for-port 2)
- ; File offset should be zero, since search-archived is false.
- (verify-nth-call-args-for 1 logviewer/find-n-matches [(first files)] 20 0 0 "search")))
- (testing "deep-search-logs-for-topology one-port search-archived = true, file-offset = 1, byte-offset = 100"
- (instrumenting
- [logviewer/find-n-matches
- logviewer/logs-for-port]
- (logviewer/deep-search-logs-for-topology "" nil topo-path "search" "20" "6700" "1" "100" true nil nil)
- (verify-call-times-for logviewer/find-n-matches 1)
- (verify-call-times-for logviewer/logs-for-port 2)
- ; File offset should be zero, since search-archived is false.
- (verify-nth-call-args-for 1 logviewer/find-n-matches files 20 1 100 "search")))
- (testing "deep-search-logs-for-topology bad-port search-archived = false, file-offset = 1"
- (instrumenting
- [logviewer/find-n-matches
- logviewer/logs-for-port]
- (logviewer/deep-search-logs-for-topology "" nil topo-path "search" "20" "2700" "1" "0" nil nil nil)
- ; Called with a bad port (not in the config) No searching should be done.
- (verify-call-times-for logviewer/find-n-matches 0)
- (verify-call-times-for logviewer/logs-for-port 0)))
- (Utils/forceDelete topo-path))))
-
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
new file mode 100644
index 0000000..612e0e2
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer;
+
+public class LogviewerConstant {
+ public static final int DEFAULT_BYTES_PER_PAGE = 51200;
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
new file mode 100644
index 0000000..2b26702
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer;
+
+import com.codahale.metrics.Meter;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.LogCleaner;
+import org.apache.storm.daemon.logviewer.webapp.LogviewerApplication;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.ui.FilterConfiguration;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Utils;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES;
+
+public class LogviewerServer implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(LogviewerServer.class);
+ private static final Meter meterShutdownCalls = StormMetricsRegistry.registerMeter("drpc:num-shutdown-calls");
+ public static final String STATIC_RESOURCE_DIRECTORY_PATH = "./public";
+
+ private static Server mkHttpServer(Map<String, Object> conf) {
+ Integer logviewerHttpPort = (Integer) conf.get(DaemonConfig.LOGVIEWER_PORT);
+ Server ret = null;
+ if (logviewerHttpPort != null && logviewerHttpPort >= 0) {
+ LOG.info("Starting Logviewer HTTP servers...");
+ Integer headerBufferSize = ObjectReader.getInt(conf.get(UI_HEADER_BUFFER_BYTES));
+ String filterClass = (String) (conf.get(DaemonConfig.UI_FILTER));
+ @SuppressWarnings("unchecked")
+ Map<String, String> filterParams = (Map<String, String>) (conf.get(DaemonConfig.UI_FILTER_PARAMS));
+ FilterConfiguration filterConfiguration = new FilterConfiguration(filterClass, filterParams);
+ final List<FilterConfiguration> filterConfigurations = Arrays.asList(filterConfiguration);
+
+ final Integer httpsPort = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_HTTPS_PORT), 0);
+ final String httpsKsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PATH));
+ final String httpsKsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PASSWORD));
+ final String httpsKsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_TYPE));
+ final String httpsKeyPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEY_PASSWORD));
+ final String httpsTsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PATH));
+ final String httpsTsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD));
+ final String httpsTsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_TYPE));
+ final Boolean httpsWantClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_WANT_CLIENT_AUTH));
+ final Boolean httpsNeedClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_NEED_CLIENT_AUTH));
+
+ //TODO a better way to do this would be great.
+ LogviewerApplication.setup(conf);
+ ret = UIHelpers.jettyCreateServer(logviewerHttpPort, null, httpsPort);
+
+ UIHelpers.configSsl(ret, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, httpsKeyPassword,
+ httpsTsPath, httpsTsPassword, httpsTsType, httpsNeedClientAuth, httpsWantClientAuth);
+
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+ try {
+ context.setBaseResource(Resource.newResource(STATIC_RESOURCE_DIRECTORY_PATH));
+ } catch (IOException e) {
+ throw new RuntimeException("Can't locate static resource directory " + STATIC_RESOURCE_DIRECTORY_PATH);
+ }
+
+ context.setContextPath("/");
+ ret.setHandler(context);
+
+ ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
+ holderPwd.setInitOrder(1);
+ context.addServlet(holderPwd,"/");
+
+ ServletHolder jerseyServlet = context.addServlet(ServletContainer.class, "/api/v1/*");
+ jerseyServlet.setInitOrder(2);
+ jerseyServlet.setInitParameter("javax.ws.rs.Application", LogviewerApplication.class.getName());
+
+ UIHelpers.configFilters(context, filterConfigurations);
+ }
+ return ret;
+ }
+
+ private final Server httpServer;
+ private boolean closed = false;
+
+ /**
+ * Constructor.
+ * @param conf Logviewer conf for the servers
+ */
+ public LogviewerServer(Map<String, Object> conf) {
+ httpServer = mkHttpServer(conf);
+ }
+
+ @VisibleForTesting
+ void start() throws Exception {
+ LOG.info("Starting Logviewer...");
+ if (httpServer != null) {
+ httpServer.start();
+ }
+ }
+
+ @VisibleForTesting
+ void awaitTermination() throws InterruptedException {
+ httpServer.join();
+ }
+
+ @Override
+ public synchronized void close() {
+ if (!closed) {
+ //This is kind of useless...
+ meterShutdownCalls.mark();
+
+ //TODO this is causing issues...
+ //if (httpServer != null) {
+ // httpServer.destroy();
+ //}
+
+ closed = true;
+ }
+ }
+
+ /**
+ * @return The port the HTTP server is listening on. Not available until {@link #start() } has run.
+ */
+ public int getHttpServerPort() {
+ assert httpServer.getConnectors().length == 1;
+
+ return httpServer.getConnectors()[0].getLocalPort();
+ }
+
+ /**
+ * Main method to start the server.
+ */
+ public static void main(String [] args) throws Exception {
+ Utils.setupDefaultUncaughtExceptionHandler();
+ Map<String, Object> conf = Utils.readStormConfig();
+
+ DirectoryCleaner directoryCleaner = new DirectoryCleaner();
+
+ try (LogviewerServer server = new LogviewerServer(conf);
+ LogCleaner logCleaner = new LogCleaner(conf, directoryCleaner)) {
+ Utils.addShutdownHookWithForceKillIn1Sec(() -> server.close());
+ logCleaner.start();
+ StormMetricsRegistry.startMetricsReporters(conf);
+ server.start();
+ server.awaitTermination();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
new file mode 100644
index 0000000..b8f32f2
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.handler;
+
+import org.apache.storm.daemon.logviewer.utils.LogFileDownloader;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+
+public class LogviewerLogDownloadHandler {
+
+ private final LogFileDownloader logFileDownloadHelper;
+
+ public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
+ this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, resourceAuthorizer);
+ }
+
+ public Response downloadLogFile(String fileName, String user) throws IOException {
+ return logFileDownloadHelper.downloadFile(fileName, user, false);
+ }
+
+ public Response downloadDaemonLogFile(String fileName, String user) throws IOException {
+ return logFileDownloadHelper.downloadFile(fileName, user, true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
new file mode 100644
index 0000000..0881b69
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.handler;
+
+import j2html.TagCreator;
+import j2html.tags.DomContent;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.LogviewerConstant;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.URLBuilder;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
+import javax.ws.rs.core.Response;
+
+import static j2html.TagCreator.*;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+
+public class LogviewerLogPageHandler {
+ private final String logRoot;
+ private final String daemonLogRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+
+ public LogviewerLogPageHandler(String logRoot, String daemonLogRoot,
+ ResourceAuthorizer resourceAuthorizer) {
+ this.logRoot = logRoot;
+ this.daemonLogRoot = daemonLogRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+ }
+
+ public Response listLogFiles(String user, Integer port, String topologyId, String callback, String origin) throws IOException {
+ List<File> fileResults = null;
+ if (topologyId == null) {
+ if (port == null) {
+ fileResults = WorkerLogs.getAllLogsForRootDir(new File(logRoot));
+ } else {
+ fileResults = new ArrayList<>();
+
+ File[] logRootFiles = new File(logRoot).listFiles();
+ if (logRootFiles != null) {
+ for (File topoDir : logRootFiles) {
+ File[] topoDirFiles = topoDir.listFiles();
+ if (topoDirFiles != null) {
+ for (File portDir : topoDirFiles) {
+ if (portDir.getName().equals(port.toString())) {
+ fileResults.addAll(DirectoryCleaner.getFilesForDir(portDir));
+ }
+ }
+ }
+ }
+ }
+ }
+ } else {
+ if (port == null) {
+ fileResults = new ArrayList<>();
+
+ File topoDir = new File(logRoot + Utils.FILE_PATH_SEPARATOR + topologyId);
+ if (topoDir.exists()) {
+ File[] topoDirFiles = topoDir.listFiles();
+ if (topoDirFiles != null) {
+ for (File portDir : topoDirFiles) {
+ fileResults.addAll(DirectoryCleaner.getFilesForDir(portDir));
+ }
+ }
+ }
+
+ } else {
+ File portDir = ConfigUtils.getWorkerDirFromRoot(logRoot, topologyId, port);
+ if (portDir.exists()) {
+ fileResults = DirectoryCleaner.getFilesForDir(portDir);
+ }
+ }
+ }
+
+ List<String> files;
+ if (fileResults != null) {
+ files = fileResults.stream()
+ .map(file -> WorkerLogs.getTopologyPortWorkerLog(file))
+ .sorted().collect(toList());
+ } else {
+ files = new ArrayList<>();
+ }
+
+ return LogviewerResponseBuilder.buildSuccessJsonResponse(files, callback, origin);
+ }
+
+ public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ String rootDir = logRoot;
+ if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ String path = file.getCanonicalPath();
+ boolean isZipFile = path.endsWith(".gz");
+ File topoDir = file.getParentFile().getParentFile();
+
+ if (file.exists() && new File(rootDir).getCanonicalFile().equals(topoDir.getParentFile())) {
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
+
+ SortedSet<File> logFiles;
+ try {
+ logFiles = Arrays.stream(topoDir.listFiles())
+ .flatMap(Unchecked.function(portDir -> DirectoryCleaner.getFilesForDir(portDir).stream()))
+ .filter(File::isFile)
+ .collect(toCollection(TreeSet::new));
+ } catch (UncheckedIOException e) {
+ throw e.getCause();
+ }
+
+ List<String> filesStrWithoutFileParam = logFiles.stream().map(WorkerLogs::getTopologyPortWorkerLog)
+ .filter(fileStr -> !StringUtils.equals(fileName, fileStr)).collect(toList());
+
+ List<String> reorderedFilesStr = new ArrayList<>();
+ reorderedFilesStr.addAll(filesStrWithoutFileParam);
+ reorderedFilesStr.add(fileName);
+
+ length = length != null ? Math.min(10485760, length) : LogviewerConstant.DEFAULT_BYTES_PER_PAGE;
+
+ String logString;
+ if (isTxtFile(fileName)) {
+ logString = escapeHtml(start != null ? pageFile(path, start, length) : pageFile(path, length));
+ } else {
+ logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
+ }
+
+ start = start != null ? start : Long.valueOf(fileLength - length).intValue();
+
+ List<DomContent> bodyContents = new ArrayList<>();
+ if (StringUtils.isNotEmpty(grep)) {
+ String matchedString = String.join("\n", Arrays.stream(logString.split("\n"))
+ .filter(str -> str.contains(grep)).collect(toList()));
+ bodyContents.add(pre(matchedString).withId("logContent"));
+ } else {
+ DomContent pagerData = null;
+ if (isTxtFile(fileName)) {
+ pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "log");
+ }
+
+ bodyContents.add(searchFileForm(fileName, "no"));
+ // list all files for this topology
+ bodyContents.add(logFileSelectionForm(reorderedFilesStr, "log"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ bodyContents.add(downloadLink(fileName));
+ bodyContents.add(pre(logString).withClass("logContent"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ }
+
+ String content = logTemplate(bodyContents, fileName, user).render();
+ return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ } else {
+ if (resourceAuthorizer.getLogUserGroupWhitelist(fileName) != null) {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ }
+ }
+
+ public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ String rootDir = daemonLogRoot;
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ String path = file.getCanonicalPath();
+ boolean isZipFile = path.endsWith(".gz");
+
+ if (file.exists() && new File(rootDir).getCanonicalFile().equals(file.getParentFile())) {
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
+
+ // all types of files included
+ List<File> logFiles = Arrays.stream(new File(rootDir).listFiles())
+ .filter(File::isFile)
+ .collect(toList());
+
+ List<String> filesStrWithoutFileParam = logFiles.stream()
+ .map(File::getName).filter(fName -> !StringUtils.equals(fileName, fName)).collect(toList());
+
+ List<String> reorderedFilesStr = new ArrayList<>();
+ reorderedFilesStr.addAll(filesStrWithoutFileParam);
+ reorderedFilesStr.add(fileName);
+
+ length = length != null ? Math.min(10485760, length) : LogviewerConstant.DEFAULT_BYTES_PER_PAGE;
+
+ String logString;
+ if (isTxtFile(fileName)) {
+ logString = escapeHtml(start != null ? pageFile(path, start, length) : pageFile(path, length));
+ } else {
+ logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
+ }
+
+ start = start != null ? start : Long.valueOf(fileLength - length).intValue();
+
+ List<DomContent> bodyContents = new ArrayList<>();
+ if (StringUtils.isNotEmpty(grep)) {
+ String matchedString = String.join("\n", Arrays.stream(logString.split("\n"))
+ .filter(str -> str.contains(grep)).collect(toList()));
+ bodyContents.add(pre(matchedString).withId("logContent"));
+ } else {
+ DomContent pagerData = null;
+ if (isTxtFile(fileName)) {
+ pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "daemonlog");
+ }
+
+ bodyContents.add(searchFileForm(fileName, "yes"));
+ // list all daemon logs
+ bodyContents.add(logFileSelectionForm(reorderedFilesStr, "daemonlog"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ bodyContents.add(daemonDownloadLink(fileName));
+ bodyContents.add(pre(logString).withClass("logContent"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ }
+
+ String content = logTemplate(bodyContents, fileName, user).render();
+ return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+ private DomContent logTemplate(List<DomContent> bodyContents, String fileName, String user) {
+ List<DomContent> finalBodyContents = new ArrayList<>();
+
+ if (StringUtils.isNotBlank(user)) {
+ finalBodyContents.add(div(p("User: " + user)).withClass("ui-user"));
+ }
+
+ finalBodyContents.add(div(p("Note: the drop-list shows at most 1024 files for each worker directory.")).withClass("ui-note"));
+ finalBodyContents.add(h3(escapeHtml(fileName)));
+ finalBodyContents.addAll(bodyContents);
+
+ return html(
+ head(
+ title(escapeHtml(fileName) + " - Storm Log Viewer"),
+ link().withRel("stylesheet").withHref("/css/bootstrap-3.3.1.min.css"),
+ link().withRel("stylesheet").withHref("/css/jquery.dataTables.1.10.4.min.css"),
+ link().withRel("stylesheet").withHref("/css/style.css")
+ ),
+ body(
+ finalBodyContents.toArray(new DomContent[]{})
+ )
+ );
+ }
+
+ private DomContent downloadLink(String fileName) {
+ return p(linkTo(UIHelpers.urlFormat("/api/v1/download?file=%s", fileName), "Download Full File"));
+ }
+
+ private DomContent daemonDownloadLink(String fileName) {
+ return p(linkTo(UIHelpers.urlFormat("/api/v1/daemondownload?file=%s", fileName), "Download Full File"));
+ }
+
+ private DomContent linkTo(String url, String content) {
+ return a(content).withHref(url);
+ }
+
+ private DomContent logFileSelectionForm(List<String> logFiles, String type) {
+ return form(
+ dropDown("file", logFiles),
+ input().withType("submit").withValue("Switch file")
+ ).withAction(type).withId("list-of-files");
+ }
+
+ private DomContent dropDown(String name, List<String> logFiles) {
+ List<DomContent> options = logFiles.stream().map(TagCreator::option).collect(toList());
+ return select(options.toArray(new DomContent[]{})).withName(name).withId(name);
+ }
+
+ private DomContent searchFileForm(String fileName, String isDaemonValue) {
+ return form(
+ text("search this file:"),
+ input().withType("text").withName("search"),
+ input().withType("hidden").withName("is-daemon").withValue(isDaemonValue),
+ input().withType("hidden").withName("file").withValue(fileName),
+ input().withType("submit").withValue("Search")
+ ).withAction("/logviewer_search.html").withId("search-box");
+ }
+
+ private DomContent pagerLinks(String fileName, Integer start, Integer length, Integer fileLength, String type) {
+ int prevStart = Math.max(0, start - length);
+ int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length;
+ List<DomContent> btnLinks = new ArrayList<>();
+
+ Map<String, Object> urlQueryParams = new HashMap<>();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("start", Math.max(0, start - length));
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start));
+
+ urlQueryParams.clear();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("start", 0);
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "First"));
+
+ urlQueryParams.clear();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Last"));
+
+ urlQueryParams.clear();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("start", Math.min(Math.max(0, fileLength - length), start + length));
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start));
+
+ return div(btnLinks.toArray(new DomContent[]{}));
+ }
+
+ private DomContent toButtonLink(String url, String text) {
+ return toButtonLink(url, text, true);
+ }
+
+ private DomContent toButtonLink(String url, String text, boolean enabled) {
+ return a(text).withHref(url).withClass("btn btn-default " + (enabled ? "enabled" : "disabled"));
+ }
+
+ private String pageFile(String path, Integer tail) throws IOException, InvalidRequestException {
+ boolean isZipFile = path.endsWith(".gz");
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(new File(path)) : new File(path).length();
+ long skip = fileLength - tail;
+ return pageFile(path, Long.valueOf(skip).intValue(), tail);
+ }
+
+ private String pageFile(String path, Integer start, Integer length) throws IOException, InvalidRequestException {
+ boolean isZipFile = path.endsWith(".gz");
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(new File(path)) : new File(path).length();
+
+ try (InputStream input = isZipFile ? new GZIPInputStream(new FileInputStream(path)) : new FileInputStream(path);
+ ByteArrayOutputStream output = new ByteArrayOutputStream()) {
+ if (start >= fileLength) {
+ throw new InvalidRequestException("Cannot start past the end of the file");
+ }
+ if (start > 0) {
+ StreamUtil.skipBytes(input, start);
+ }
+
+ byte[] buffer = new byte[1024];
+ while (output.size() < length) {
+ int size = input.read(buffer, 0, Math.min(1024, length - output.size()));
+ if (size > 0) {
+ output.write(buffer, 0, size);
+ } else {
+ break;
+ }
+ }
+
+ return output.toString();
+ }
+ }
+
+ private boolean isTxtFile(String fileName) {
+ Pattern p = Pattern.compile("\\.(log.*|txt|yaml|pid)$");
+ Matcher matcher = p.matcher(fileName);
+ return matcher.find();
+ }
+}
[05/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
new file mode 100644
index 0000000..bc6529c
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.webapp;
+
+import com.codahale.metrics.Meter;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.ui.UIHelpers;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.Map;
+
+@Path("/")
+public class LogviewerResource {
+ private static final Logger LOG = LoggerFactory.getLogger(LogviewerResource.class);
+
+ private static final Meter meterLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-log-page-http-requests");
+ private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-daemonlog-page-http-requests");
+ private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-file-http-requests");
+ private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-daemon-file-http-requests");
+ private static final Meter meterListLogsHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-list-logs-http-requests");
+
+ private final LogviewerLogPageHandler logviewer;
+ private final LogviewerProfileHandler profileHandler;
+ private final LogviewerLogDownloadHandler logDownloadHandler;
+ private final LogviewerLogSearchHandler logSearchHandler;
+ private final IHttpCredentialsPlugin httpCredsHandler;
+
+ public LogviewerResource(LogviewerLogPageHandler logviewerParam, LogviewerProfileHandler profileHandler,
+ LogviewerLogDownloadHandler logDownloadHandler, LogviewerLogSearchHandler logSearchHandler,
+ IHttpCredentialsPlugin httpCredsHandler) {
+ this.logviewer = logviewerParam;
+ this.profileHandler = profileHandler;
+ this.logDownloadHandler = logDownloadHandler;
+ this.logSearchHandler = logSearchHandler;
+ this.httpCredsHandler = httpCredsHandler;
+ }
+
+ @GET
+ @Path("/log")
+ public Response log(@Context HttpServletRequest request) throws IOException {
+ meterLogPageHttpRequests.mark();
+
+ try {
+ String user = httpCredsHandler.getUserName(request);
+ Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null;
+ Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null;
+ String decodedFileName = URLDecoder.decode(request.getParameter("file"));
+ String grep = request.getParameter("grep");
+ return logviewer.logPage(decodedFileName, start, length, grep, user);
+ } catch (InvalidRequestException e) {
+ LOG.error(e.getMessage(), e);
+ return Response.status(400).entity(e.getMessage()).build();
+ }
+ }
+
+ @GET
+ @Path("/daemonlog")
+ public Response daemonLog(@Context HttpServletRequest request) throws IOException {
+ meterDaemonLogPageHttpRequests.mark();
+
+ try {
+ String user = httpCredsHandler.getUserName(request);
+ Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null;
+ Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null;
+ String decodedFileName = URLDecoder.decode(request.getParameter("file"));
+ String grep = request.getParameter("grep");
+ return logviewer.daemonLogPage(decodedFileName, start, length, grep, user);
+ } catch (InvalidRequestException e) {
+ LOG.error(e.getMessage(), e);
+ return Response.status(400).entity(e.getMessage()).build();
+ }
+ }
+
+ @GET
+ @Path("/searchLogs")
+ public Response searchLogs(@Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ String topologyId = request.getParameter("topoId");
+ String portStr = request.getParameter("port");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
+ }
+
+ @GET
+ @Path("/listLogs")
+ public Response listLogs(@Context HttpServletRequest request) throws IOException {
+ meterListLogsHttpRequests.mark();
+
+ String user = httpCredsHandler.getUserName(request);
+ String topologyId = request.getParameter("topoId");
+ String portStr = request.getParameter("port");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
+ }
+
+ @GET
+ @Path("/dumps/{topo-id}/{host-port}")
+ public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
+ @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ return profileHandler.listDumpFiles(topologyId, hostPort, user);
+ }
+
+ @GET
+ @Path("/dumps/{topo-id}/{host-port}/{filename}")
+ public Response downloadDumpFile(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
+ @PathParam("filename") String fileName, @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ return profileHandler.downloadDumpFile(topologyId, hostPort, fileName, user);
+ }
+
+ @GET
+ @Path("/download")
+ public Response downloadLogFile(@Context HttpServletRequest request) throws IOException {
+ meterDownloadLogFileHttpRequests.mark();
+
+ String user = httpCredsHandler.getUserName(request);
+ String file = request.getParameter("file");
+ String decodedFileName = URLDecoder.decode(file);
+ return logDownloadHandler.downloadLogFile(decodedFileName, user);
+ }
+
+ @GET
+ @Path("/daemondownload")
+ public Response downloadDaemonLogFile(@Context HttpServletRequest request) throws IOException {
+ meterDownloadLogDaemonFileHttpRequests.mark();
+
+ String user = httpCredsHandler.getUserName(request);
+ String file = request.getParameter("file");
+ String decodedFileName = URLDecoder.decode(file);
+ return logDownloadHandler.downloadDaemonLogFile(decodedFileName, user);
+ }
+
+ @GET
+ @Path("/search/{file}")
+ public Response search(@PathParam("file") String file, @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ boolean isDaemon = StringUtils.equals(request.getParameter("is-daemon"), "yes");
+ String decodedFileName = URLDecoder.decode(file);
+ String searchString = request.getParameter("search-string");
+ String numMatchesStr = request.getParameter("num-matches");
+ String startByteOffset = request.getParameter("start-byte-offset");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ try {
+ return logSearchHandler.searchLogFile(decodedFileName, user, isDaemon, searchString, numMatchesStr,
+ startByteOffset, callback, origin);
+ } catch (InvalidRequestException e) {
+ LOG.error(e.getMessage(), e);
+ return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(e)).setCallback(callback)
+ .setStatus(400).build();
+ }
+ }
+
+ @GET
+ @Path("/deepSearch/{topoId}")
+ public Response deepSearch(@PathParam("topoId") String topologyId,
+ @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ String searchString = request.getParameter("search-string");
+ String numMatchesStr = request.getParameter("num-matches");
+ String portStr = request.getParameter("port");
+ String startFileOffset = request.getParameter("start-file-offset");
+ String startByteOffset = request.getParameter("start-byte-offset");
+ String searchArchived = request.getParameter("search-archived");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ return logSearchHandler.deepSearchLogsForTopology(topologyId, user, searchString, numMatchesStr, portStr,
+ startFileOffset, startByteOffset, BooleanUtils.toBooleanObject(searchArchived), callback, origin);
+ }
+
+ private int parseIntegerFromMap(Map map, String parameterKey) throws InvalidRequestException {
+ try {
+ return Integer.parseInt(((String[]) map.get(parameterKey))[0]);
+ } catch (NumberFormatException ex) {
+ throw new InvalidRequestException("Could not make an integer out of the query parameter '"
+ + parameterKey + "'", ex);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java
deleted file mode 100644
index 01cc0bc..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer;
-
-public class LogviewerConstant {
- public static final int DEFAULT_BYTES_PER_PAGE = 51200;
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java
deleted file mode 100644
index 24ccf6e..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer;
-
-import com.codahale.metrics.Meter;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.daemon.drpc.webapp.ReqContextFilter;
-import org.apache.storm.daemon.wip.logviewer.utils.LogCleaner;
-import org.apache.storm.daemon.wip.logviewer.webapp.LogviewerApplication;
-import org.apache.storm.metric.StormMetricsRegistry;
-import org.apache.storm.security.auth.AuthUtils;
-import org.apache.storm.security.auth.IHttpCredentialsPlugin;
-import org.apache.storm.ui.FilterConfiguration;
-import org.apache.storm.ui.UIHelpers;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.Utils;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.DefaultServlet;
-import org.eclipse.jetty.servlet.FilterHolder;
-import org.eclipse.jetty.servlet.FilterMapping;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.util.resource.Resource;
-import org.glassfish.jersey.servlet.ServletContainer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES;
-
-public class LogviewerServer implements AutoCloseable {
- private static final Logger LOG = LoggerFactory.getLogger(LogviewerServer.class);
- private static final Meter meterShutdownCalls = StormMetricsRegistry.registerMeter("drpc:num-shutdown-calls");
- public static final String STATIC_RESOURCE_DIRECTORY_PATH = "./public";
-
- private static Server mkHttpServer(Map<String, Object> conf) {
- Integer logviewerHttpPort = (Integer) conf.get(DaemonConfig.LOGVIEWER_PORT);
- Server ret = null;
- if (logviewerHttpPort != null && logviewerHttpPort >= 0) {
- LOG.info("Starting Logviewer HTTP servers...");
- Integer headerBufferSize = ObjectReader.getInt(conf.get(UI_HEADER_BUFFER_BYTES));
- String filterClass = (String) (conf.get(DaemonConfig.UI_FILTER));
- @SuppressWarnings("unchecked")
- Map<String, String> filterParams = (Map<String, String>) (conf.get(DaemonConfig.UI_FILTER_PARAMS));
- FilterConfiguration filterConfiguration = new FilterConfiguration(filterClass, filterParams);
- final List<FilterConfiguration> filterConfigurations = Arrays.asList(filterConfiguration);
-
- final Integer httpsPort = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_HTTPS_PORT), 0);
- final String httpsKsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PATH));
- final String httpsKsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PASSWORD));
- final String httpsKsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_TYPE));
- final String httpsKeyPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEY_PASSWORD));
- final String httpsTsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PATH));
- final String httpsTsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD));
- final String httpsTsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_TYPE));
- final Boolean httpsWantClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_WANT_CLIENT_AUTH));
- final Boolean httpsNeedClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_NEED_CLIENT_AUTH));
-
- //TODO a better way to do this would be great.
- LogviewerApplication.setup(conf);
- ret = UIHelpers.jettyCreateServer(logviewerHttpPort, null, httpsPort);
-
- UIHelpers.configSsl(ret, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, httpsKeyPassword,
- httpsTsPath, httpsTsPassword, httpsTsType, httpsNeedClientAuth, httpsWantClientAuth);
-
- ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
- try {
- context.setBaseResource(Resource.newResource(STATIC_RESOURCE_DIRECTORY_PATH));
- } catch (IOException e) {
- throw new RuntimeException("Can't locate static resource directory " + STATIC_RESOURCE_DIRECTORY_PATH);
- }
-
- context.setContextPath("/");
- ret.setHandler(context);
-
- ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
- holderPwd.setInitOrder(1);
- context.addServlet(holderPwd,"/");
-
- ServletHolder jerseyServlet = context.addServlet(ServletContainer.class, "/api/v1/*");
- jerseyServlet.setInitOrder(2);
- jerseyServlet.setInitParameter("javax.ws.rs.Application", LogviewerApplication.class.getName());
-
- UIHelpers.configFilters(context, filterConfigurations);
- }
- return ret;
- }
-
- private final Server httpServer;
- private boolean closed = false;
-
- /**
- * Constructor.
- * @param conf Logviewer conf for the servers
- */
- public LogviewerServer(Map<String, Object> conf) {
- httpServer = mkHttpServer(conf);
- }
-
- @VisibleForTesting
- void start() throws Exception {
- LOG.info("Starting Logviewer...");
- if (httpServer != null) {
- httpServer.start();
- }
- }
-
- @VisibleForTesting
- void awaitTermination() throws InterruptedException {
- httpServer.join();
- }
-
- @Override
- public synchronized void close() {
- if (!closed) {
- //This is kind of useless...
- meterShutdownCalls.mark();
-
- //TODO this is causing issues...
- //if (httpServer != null) {
- // httpServer.destroy();
- //}
-
- closed = true;
- }
- }
-
- /**
- * @return The port the HTTP server is listening on. Not available until {@link #start() } has run.
- */
- public int getHttpServerPort() {
- assert httpServer.getConnectors().length == 1;
-
- return httpServer.getConnectors()[0].getLocalPort();
- }
-
- /**
- * Main method to start the server.
- */
- public static void main(String [] args) throws Exception {
- Utils.setupDefaultUncaughtExceptionHandler();
- Map<String, Object> conf = Utils.readStormConfig();
-
- try (LogviewerServer server = new LogviewerServer(conf);
- LogCleaner logCleaner = new LogCleaner(conf)) {
- Utils.addShutdownHookWithForceKillIn1Sec(() -> server.close());
- logCleaner.start();
- StormMetricsRegistry.startMetricsReporters(conf);
- server.start();
- server.awaitTermination();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java
deleted file mode 100644
index 26d1d63..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.handler;
-
-import org.apache.storm.daemon.wip.logviewer.utils.LogFileDownloader;
-import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
-
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-
-public class LogviewerLogDownloadHandler {
-
- private final LogFileDownloader logFileDownloadHelper;
-
- public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
- this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, resourceAuthorizer);
- }
-
- public Response downloadLogFile(String fileName, String user) throws IOException {
- return logFileDownloadHelper.downloadFile(fileName, user, false);
- }
-
- public Response downloadDaemonLogFile(String fileName, String user) throws IOException {
- return logFileDownloadHelper.downloadFile(fileName, user, true);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java
deleted file mode 100644
index 0e623f5..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.handler;
-
-import j2html.TagCreator;
-import j2html.tags.DomContent;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.daemon.DirectoryCleaner;
-import org.apache.storm.daemon.utils.StreamUtil;
-import org.apache.storm.daemon.utils.URLBuilder;
-import org.apache.storm.daemon.wip.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.wip.logviewer.utils.WorkerLogs;
-import org.apache.storm.ui.InvalidRequestException;
-import org.apache.storm.ui.UIHelpers;
-import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ServerUtils;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UncheckedIOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
-
-import javax.ws.rs.core.Response;
-
-import static j2html.TagCreator.*;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toList;
-import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
-import static org.apache.storm.daemon.wip.logviewer.LogviewerConstant.DEFAULT_BYTES_PER_PAGE;
-
-public class LogviewerLogPageHandler {
- private final String logRoot;
- private final String daemonLogRoot;
- private final ResourceAuthorizer resourceAuthorizer;
-
- public LogviewerLogPageHandler(String logRoot, String daemonLogRoot,
- ResourceAuthorizer resourceAuthorizer) {
- this.logRoot = logRoot;
- this.daemonLogRoot = daemonLogRoot;
- this.resourceAuthorizer = resourceAuthorizer;
- }
-
- public Response listLogFiles(String user, Integer port, String topologyId, String callback, String origin) throws IOException {
- List<File> fileResults = null;
- if (topologyId == null) {
- if (port == null) {
- fileResults = WorkerLogs.getAllLogsForRootDir(new File(logRoot));
- } else {
- fileResults = new ArrayList<>();
-
- File[] logRootFiles = new File(logRoot).listFiles();
- if (logRootFiles != null) {
- for (File topoDir : logRootFiles) {
- File[] topoDirFiles = topoDir.listFiles();
- if (topoDirFiles != null) {
- for (File portDir : topoDirFiles) {
- if (portDir.getName().equals(port.toString())) {
- fileResults.addAll(DirectoryCleaner.getFilesForDir(portDir));
- }
- }
- }
- }
- }
- }
- } else {
- if (port == null) {
- fileResults = new ArrayList<>();
-
- File topoDir = new File(logRoot + Utils.FILE_PATH_SEPARATOR + topologyId);
- if (topoDir.exists()) {
- File[] topoDirFiles = topoDir.listFiles();
- if (topoDirFiles != null) {
- for (File portDir : topoDirFiles) {
- fileResults.addAll(DirectoryCleaner.getFilesForDir(portDir));
- }
- }
- }
-
- } else {
- File portDir = ConfigUtils.getWorkerDirFromRoot(logRoot, topologyId, port);
- if (portDir.exists()) {
- fileResults = DirectoryCleaner.getFilesForDir(portDir);
- }
- }
- }
-
- List<String> files;
- if (fileResults != null) {
- files = fileResults.stream()
- .map(file -> WorkerLogs.getTopologyPortWorkerLog(file))
- .sorted().collect(toList());
- } else {
- files = new ArrayList<>();
- }
-
- return LogviewerResponseBuilder.buildSuccessJsonResponse(files, callback, origin);
- }
-
- public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
- String rootDir = logRoot;
- if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
- File file = new File(rootDir, fileName).getCanonicalFile();
- String path = file.getCanonicalPath();
- boolean isZipFile = path.endsWith(".gz");
- File topoDir = file.getParentFile().getParentFile();
-
- if (file.exists() && new File(rootDir).getCanonicalFile().equals(topoDir.getParentFile())) {
- long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
-
- SortedSet<File> logFiles;
- try {
- logFiles = Arrays.stream(topoDir.listFiles())
- .flatMap(Unchecked.function(portDir -> DirectoryCleaner.getFilesForDir(portDir).stream()))
- .filter(File::isFile)
- .collect(toCollection(TreeSet::new));
- } catch (UncheckedIOException e) {
- throw e.getCause();
- }
-
- List<String> filesStrWithoutFileParam = logFiles.stream().map(WorkerLogs::getTopologyPortWorkerLog)
- .filter(fileStr -> !StringUtils.equals(fileName, fileStr)).collect(toList());
-
- List<String> reorderedFilesStr = new ArrayList<>();
- reorderedFilesStr.addAll(filesStrWithoutFileParam);
- reorderedFilesStr.add(fileName);
-
- length = length != null ? Math.min(10485760, length) : DEFAULT_BYTES_PER_PAGE;
-
- String logString;
- if (isTxtFile(fileName)) {
- logString = escapeHtml(start != null ? pageFile(path, start, length) : pageFile(path, length));
- } else {
- logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
- }
-
- start = start != null ? start : Long.valueOf(fileLength - length).intValue();
-
- List<DomContent> bodyContents = new ArrayList<>();
- if (StringUtils.isNotEmpty(grep)) {
- String matchedString = String.join("\n", Arrays.stream(logString.split("\n"))
- .filter(str -> str.contains(grep)).collect(toList()));
- bodyContents.add(pre(matchedString).withId("logContent"));
- } else {
- DomContent pagerData = null;
- if (isTxtFile(fileName)) {
- pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "log");
- }
-
- bodyContents.add(searchFileForm(fileName, "no"));
- // list all files for this topology
- bodyContents.add(logFileSelectionForm(reorderedFilesStr, "log"));
- if (pagerData != null) {
- bodyContents.add(pagerData);
- }
- bodyContents.add(downloadLink(fileName));
- bodyContents.add(pre(logString).withClass("logContent"));
- if (pagerData != null) {
- bodyContents.add(pagerData);
- }
- }
-
- String content = logTemplate(bodyContents, fileName, user).render();
- return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
- } else {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- }
- } else {
- if (resourceAuthorizer.getLogUserGroupWhitelist(fileName) != null) {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- } else {
- return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
- }
- }
- }
-
- public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
- String rootDir = daemonLogRoot;
- File file = new File(rootDir, fileName).getCanonicalFile();
- String path = file.getCanonicalPath();
- boolean isZipFile = path.endsWith(".gz");
-
- if (file.exists() && new File(rootDir).getCanonicalFile().equals(file.getParentFile())) {
- long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
-
- // all types of files included
- List<File> logFiles = Arrays.stream(new File(rootDir).listFiles())
- .filter(File::isFile)
- .collect(toList());
-
- List<String> filesStrWithoutFileParam = logFiles.stream()
- .map(File::getName).filter(fName -> !StringUtils.equals(fileName, fName)).collect(toList());
-
- List<String> reorderedFilesStr = new ArrayList<>();
- reorderedFilesStr.addAll(filesStrWithoutFileParam);
- reorderedFilesStr.add(fileName);
-
- length = length != null ? Math.min(10485760, length) : DEFAULT_BYTES_PER_PAGE;
-
- String logString;
- if (isTxtFile(fileName)) {
- logString = escapeHtml(start != null ? pageFile(path, start, length) : pageFile(path, length));
- } else {
- logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
- }
-
- start = start != null ? start : Long.valueOf(fileLength - length).intValue();
-
- List<DomContent> bodyContents = new ArrayList<>();
- if (StringUtils.isNotEmpty(grep)) {
- String matchedString = String.join("\n", Arrays.stream(logString.split("\n"))
- .filter(str -> str.contains(grep)).collect(toList()));
- bodyContents.add(pre(matchedString).withId("logContent"));
- } else {
- DomContent pagerData = null;
- if (isTxtFile(fileName)) {
- pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "daemonlog");
- }
-
- bodyContents.add(searchFileForm(fileName, "yes"));
- // list all daemon logs
- bodyContents.add(logFileSelectionForm(reorderedFilesStr, "daemonlog"));
- if (pagerData != null) {
- bodyContents.add(pagerData);
- }
- bodyContents.add(daemonDownloadLink(fileName));
- bodyContents.add(pre(logString).withClass("logContent"));
- if (pagerData != null) {
- bodyContents.add(pagerData);
- }
- }
-
- String content = logTemplate(bodyContents, fileName, user).render();
- return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
- } else {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- }
- }
-
- private DomContent logTemplate(List<DomContent> bodyContents, String fileName, String user) {
- List<DomContent> finalBodyContents = new ArrayList<>();
-
- if (StringUtils.isNotBlank(user)) {
- finalBodyContents.add(div(p("User: " + user)).withClass("ui-user"));
- }
-
- finalBodyContents.add(div(p("Note: the drop-list shows at most 1024 files for each worker directory.")).withClass("ui-note"));
- finalBodyContents.add(h3(escapeHtml(fileName)));
- finalBodyContents.addAll(bodyContents);
-
- return html(
- head(
- title(escapeHtml(fileName) + " - Storm Log Viewer"),
- link().withRel("stylesheet").withHref("/css/bootstrap-3.3.1.min.css"),
- link().withRel("stylesheet").withHref("/css/jquery.dataTables.1.10.4.min.css"),
- link().withRel("stylesheet").withHref("/css/style.css")
- ),
- body(
- finalBodyContents.toArray(new DomContent[]{})
- )
- );
- }
-
- private DomContent downloadLink(String fileName) {
- return p(linkTo(UIHelpers.urlFormat("/api/v1/download?file=%s", fileName), "Download Full File"));
- }
-
- private DomContent daemonDownloadLink(String fileName) {
- return p(linkTo(UIHelpers.urlFormat("/api/v1/daemondownload?file=%s", fileName), "Download Full File"));
- }
-
- private DomContent linkTo(String url, String content) {
- return a(content).withHref(url);
- }
-
- private DomContent logFileSelectionForm(List<String> logFiles, String type) {
- return form(
- dropDown("file", logFiles),
- input().withType("submit").withValue("Switch file")
- ).withAction(type).withId("list-of-files");
- }
-
- private DomContent dropDown(String name, List<String> logFiles) {
- List<DomContent> options = logFiles.stream().map(TagCreator::option).collect(toList());
- return select(options.toArray(new DomContent[]{})).withName(name).withId(name);
- }
-
- private DomContent searchFileForm(String fileName, String isDaemonValue) {
- return form(
- text("search this file:"),
- input().withType("text").withName("search"),
- input().withType("hidden").withName("is-daemon").withValue(isDaemonValue),
- input().withType("hidden").withName("file").withValue(fileName),
- input().withType("submit").withValue("Search")
- ).withAction("/logviewer_search.html").withId("search-box");
- }
-
- private DomContent pagerLinks(String fileName, Integer start, Integer length, Integer fileLength, String type) {
- int prevStart = Math.max(0, start - length);
- int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length;
- List<DomContent> btnLinks = new ArrayList<>();
-
- Map<String, Object> urlQueryParams = new HashMap<>();
- urlQueryParams.put("file", fileName);
- urlQueryParams.put("start", Math.max(0, start - length));
- urlQueryParams.put("length", length);
-
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start));
-
- urlQueryParams.clear();
- urlQueryParams.put("file", fileName);
- urlQueryParams.put("start", 0);
- urlQueryParams.put("length", length);
-
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "First"));
-
- urlQueryParams.clear();
- urlQueryParams.put("file", fileName);
- urlQueryParams.put("length", length);
-
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Last"));
-
- urlQueryParams.clear();
- urlQueryParams.put("file", fileName);
- urlQueryParams.put("start", Math.min(Math.max(0, fileLength - length), start + length));
- urlQueryParams.put("length", length);
-
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start));
-
- return div(btnLinks.toArray(new DomContent[]{}));
- }
-
- private DomContent toButtonLink(String url, String text) {
- return toButtonLink(url, text, true);
- }
-
- private DomContent toButtonLink(String url, String text, boolean enabled) {
- return a(text).withHref(url).withClass("btn btn-default " + (enabled ? "enabled" : "disabled"));
- }
-
- private String pageFile(String path, Integer tail) throws IOException, InvalidRequestException {
- boolean isZipFile = path.endsWith(".gz");
- long fileLength = isZipFile ? ServerUtils.zipFileSize(new File(path)) : new File(path).length();
- long skip = fileLength - tail;
- return pageFile(path, Long.valueOf(skip).intValue(), tail);
- }
-
- private String pageFile(String path, Integer start, Integer length) throws IOException, InvalidRequestException {
- boolean isZipFile = path.endsWith(".gz");
- long fileLength = isZipFile ? ServerUtils.zipFileSize(new File(path)) : new File(path).length();
-
- try (InputStream input = isZipFile ? new GZIPInputStream(new FileInputStream(path)) : new FileInputStream(path);
- ByteArrayOutputStream output = new ByteArrayOutputStream()) {
- if (start >= fileLength) {
- throw new InvalidRequestException("Cannot start past the end of the file");
- }
- if (start > 0) {
- StreamUtil.skipBytes(input, start);
- }
-
- byte[] buffer = new byte[1024];
- while (output.size() < length) {
- int size = input.read(buffer, 0, Math.min(1024, length - output.size()));
- if (size > 0) {
- output.write(buffer, 0, size);
- } else {
- break;
- }
- }
-
- return output.toString();
- }
- }
-
- private boolean isTxtFile(String fileName) {
- Pattern p = Pattern.compile("\\.(log.*|txt|yaml|pid)$");
- Matcher matcher = p.matcher(fileName);
- return matcher.find();
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java
deleted file mode 100644
index c0a69cb..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java
+++ /dev/null
@@ -1,686 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.handler;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.BooleanUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.daemon.DirectoryCleaner;
-import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.daemon.utils.StreamUtil;
-import org.apache.storm.daemon.utils.URLBuilder;
-import org.apache.storm.daemon.wip.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.wip.logviewer.utils.WorkerLogs;
-import org.apache.storm.ui.InvalidRequestException;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ServerUtils;
-import org.apache.storm.utils.Utils;
-import org.json.simple.JSONAware;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.Response;
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPInputStream;
-
-import static java.util.stream.Collectors.toList;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.first;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.last;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
-import static org.apache.storm.daemon.wip.logviewer.LogviewerConstant.DEFAULT_BYTES_PER_PAGE;
-
-public class LogviewerLogSearchHandler {
- private final static Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class);
-
- public static final int GREP_MAX_SEARCH_SIZE = 1024;
- public static final int GREP_BUF_SIZE = 2048;
- public static final int GREP_CONTEXT_SIZE = 128;
- public static final Pattern WORKER_LOG_FILENAME_PATTERN = Pattern.compile("^worker.log(.*)");
-
- private final Map<String, Object> stormConf;
- private final String logRoot;
- private final String daemonLogRoot;
- private final ResourceAuthorizer resourceAuthorizer;
- private final Integer logviewerPort;
-
- public LogviewerLogSearchHandler(Map<String, Object> stormConf, String logRoot, String daemonLogRoot,
- ResourceAuthorizer resourceAuthorizer) {
- this.stormConf = stormConf;
- this.logRoot = logRoot;
- this.daemonLogRoot = daemonLogRoot;
- this.resourceAuthorizer = resourceAuthorizer;
-
- this.logviewerPort = ObjectReader.getInt(stormConf.get(DaemonConfig.LOGVIEWER_PORT));
- }
-
- public Response searchLogFile(String fileName, String user, boolean isDaemon, String search,
- String numMatchesStr, String offsetStr, String callback, String origin)
- throws IOException, InvalidRequestException {
- String rootDir = isDaemon ? daemonLogRoot : logRoot;
- File file = new File(rootDir, fileName).getCanonicalFile();
- Response response;
- if (file.exists()) {
- if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
- Integer numMatchesInt = numMatchesStr != null ? tryParseIntParam("num-matches", numMatchesStr) : null;
- Integer offsetInt = offsetStr != null ? tryParseIntParam("start-byte-offset", offsetStr) : null;
-
- try {
- if (StringUtils.isNotEmpty(search) && search.getBytes("UTF-8").length <= GREP_MAX_SEARCH_SIZE) {
- Map<String, Object> entity = new HashMap<>();
- entity.put("isDaemon", isDaemon ? "yes" : "no");
- entity.putAll(substringSearch(file, search, isDaemon, numMatchesInt, offsetInt));
-
- response = LogviewerResponseBuilder.buildSuccessJsonResponse(entity, callback, origin);
- } else {
- throw new InvalidRequestException("Search substring must be between 1 and 1024 "
- + "UTF-8 bytes in size (inclusive)");
- }
- } catch (Exception ex) {
- response = LogviewerResponseBuilder.buildExceptionJsonResponse(ex, callback);
- }
- } else {
- // unauthorized
- response = LogviewerResponseBuilder.buildUnauthorizedUserJsonResponse(user, callback);
- }
- } else {
- // not found
- Map<String, String> entity = new HashMap<>();
- entity.put("error", "Not Found");
- entity.put("errorMessage", "The file was not found on this node.");
-
- response = new JsonResponseBuilder().setData(entity).setCallback(callback).setStatus(404).build();
- }
-
- return response;
- }
-
- public Response deepSearchLogsForTopology(String topologyId, String user, String search,
- String numMatchesStr, String portStr, String fileOffsetStr, String offsetStr,
- Boolean searchArchived, String callback, String origin) {
- String rootDir = logRoot;
- Object returnValue;
- File topologyDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId);
- if (StringUtils.isEmpty(search) || !topologyDir.exists()) {
- returnValue = new ArrayList<>();
- } else {
- int fileOffset = ObjectReader.getInt(fileOffsetStr, 0);
- int offset = ObjectReader.getInt(offsetStr, 0);
- int numMatches = ObjectReader.getInt(numMatchesStr, 1);
-
- File[] portDirsArray = topologyDir.listFiles();
- List<File> portDirs;
- if (portDirsArray != null) {
- portDirs = Arrays.asList(portDirsArray);
- } else {
- portDirs = new ArrayList<>();
- }
-
- if (StringUtils.isEmpty(portStr) || portStr.equals("*")) {
- // check for all ports
- List<List<File>> filteredLogs = portDirs.stream()
- .map(portDir -> logsForPort(user, portDir))
- .filter(logs -> logs != null && !logs.isEmpty())
- .collect(toList());
-
- if (BooleanUtils.isTrue(searchArchived)) {
- returnValue = filteredLogs.stream()
- .map(fl -> findNMatches(fl, numMatches, 0, 0, search))
- .collect(toList());
- } else {
- returnValue = filteredLogs.stream()
- .map(fl -> Collections.singletonList(first(fl)))
- .map(fl -> findNMatches(fl, numMatches, 0, 0, search))
- .collect(toList());
- }
- } else {
- int port = Integer.parseInt(portStr);
- // check just the one port
- List<Integer> slotsPorts = (List<Integer>) stormConf.getOrDefault(DaemonConfig.SUPERVISOR_SLOTS_PORTS,
- new ArrayList<>());
- boolean containsPort = slotsPorts.stream()
- .anyMatch(slotPort -> slotPort != null && (slotPort == port));
- if (!containsPort) {
- returnValue = new ArrayList<>();
- } else {
- File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId +
- Utils.FILE_PATH_SEPARATOR + port);
-
- if (!portDir.exists() || logsForPort(user, portDir).isEmpty()) {
- returnValue = new ArrayList<>();
- } else {
- List<File> filteredLogs = logsForPort(user, portDir);
- if (BooleanUtils.isTrue(searchArchived)) {
- returnValue = findNMatches(filteredLogs, numMatches, fileOffset, offset, search);
- } else {
- returnValue = findNMatches(Collections.singletonList(first(filteredLogs)),
- numMatches, 0, offset, search);
- }
- }
- }
- }
- }
-
- return LogviewerResponseBuilder.buildSuccessJsonResponse(returnValue, callback, origin);
- }
-
- private Integer tryParseIntParam(String paramName, String value) throws InvalidRequestException {
- try {
- return Integer.parseInt(value);
- } catch (NumberFormatException e) {
- throw new InvalidRequestException("Could not parse " + paramName + " to an integer");
- }
- }
-
- private Map<String,Object> substringSearch(File file, String searchString, int numMatches, int startByteOffset) throws InvalidRequestException {
- return substringSearch(file, searchString, false, numMatches, startByteOffset);
- }
-
- private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches, Integer startByteOffset) throws InvalidRequestException {
- try {
- if (StringUtils.isEmpty(searchString)) {
- throw new IllegalArgumentException("Precondition fails: search string should not be empty.");
- }
- if (searchString.getBytes("UTF-8").length > GREP_MAX_SEARCH_SIZE) {
- throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + GREP_MAX_SEARCH_SIZE);
- }
-
- boolean isZipFile = file.getName().endsWith(".gz");
- FileInputStream fis = new FileInputStream(file);
- InputStream gzippedInputStream;
- if (isZipFile) {
- gzippedInputStream = new GZIPInputStream(fis);
- } else {
- gzippedInputStream = fis;
- }
-
- BufferedInputStream stream = new BufferedInputStream(gzippedInputStream);
-
- int fileLength;
- if (isZipFile) {
- fileLength = (int) ServerUtils.zipFileSize(file);
- } else {
- fileLength = (int) file.length();
- }
-
- ByteBuffer buf = ByteBuffer.allocate(GREP_BUF_SIZE);
- byte[] bufArray = buf.array();
- int totalBytesRead = 0;
- byte[] searchBytes = searchString.getBytes("UTF-8");
- numMatches = numMatches != null ? numMatches : 10;
- startByteOffset = startByteOffset != null ? startByteOffset : 0;
-
- // Start at the part of the log file we are interested in.
- // Allow searching when start-byte-offset == file-len so it doesn't blow up on 0-length files
- if (startByteOffset > fileLength) {
- throw new InvalidRequestException("Cannot search past the end of the file");
- }
-
- if (startByteOffset > 0) {
- StreamUtil.skipBytes(stream, startByteOffset);
- }
-
- Arrays.fill(bufArray, (byte) 0);
-
- int bytesRead = stream.read(bufArray, 0, Math.min((int) fileLength, GREP_BUF_SIZE));
- buf.limit(bytesRead);
- totalBytesRead += bytesRead;
-
- List<Map<String, Object>> initialMatches = new ArrayList<>();
- int initBufOffset = 0;
- int byteOffset = startByteOffset;
- byte[] beforeBytes = null;
-
- Map<String, Object> ret = new HashMap<>();
- while (true) {
- SubstringSearchResult searchRet = bufferSubstringSearch(isDaemon, file, fileLength, byteOffset, initBufOffset,
- stream, startByteOffset, totalBytesRead, buf, searchBytes, initialMatches, numMatches, beforeBytes);
-
- List<Map<String, Object>> matches = searchRet.getMatches();
- Integer newByteOffset = searchRet.getNewByteOffset();
- byte[] newBeforeBytes = searchRet.getNewBeforeBytes();
-
- if (matches.size() < numMatches && totalBytesRead + startByteOffset < fileLength) {
- // The start index is positioned to find any possible
- // occurrence search string that did not quite fit in the
- // buffer on the previous read.
- int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length;
-
- totalBytesRead = rotateGrepBuffer(buf, stream, totalBytesRead, file, fileLength);
- if (totalBytesRead < 0) {
- throw new InvalidRequestException("Cannot search past the end of the file");
- }
-
- initialMatches = matches;
- initBufOffset = newBufOffset;
- byteOffset = newByteOffset;
- beforeBytes = newBeforeBytes;
- } else {
- ret.put("isDaemon", isDaemon ? "yes" : "no");
- Integer nextByteOffset = null;
- if (matches.size() >= numMatches || totalBytesRead < fileLength) {
- nextByteOffset = (Integer) last(matches).get("byteOffset") + searchBytes.length;
- if (fileLength <= nextByteOffset) {
- nextByteOffset = null;
- }
- }
- ret.putAll(mkGrepResponse(searchBytes, startByteOffset, matches, nextByteOffset));
- break;
- }
- }
-
- return ret;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-
- /**
- * Get the filtered, authorized, sorted log files for a port.
- */
- private List<File> logsForPort(String user, File portDir) {
- try {
- List<File> workerLogs = DirectoryCleaner.getFilesForDir(portDir).stream()
- .filter(file -> WORKER_LOG_FILENAME_PATTERN.asPredicate().test(file.getName()))
- .collect(toList());
-
- return workerLogs.stream()
- .filter(log -> resourceAuthorizer.isUserAllowedToAccessFile(user, WorkerLogs.getTopologyPortWorkerLog(log)))
- .sorted((f1, f2) -> (int) (f2.lastModified() - f1.lastModified()))
- .collect(toList());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private Matched findNMatches(List<File> logs, int numMatches, int fileOffset, int offset, String search) {
- logs = drop(logs, fileOffset);
-
- List<Map<String, Object>> matches = new ArrayList<>();
- int matchCount = 0;
-
- while (true) {
- if (logs.isEmpty()) {
- break;
- }
-
- File firstLog = logs.get(0);
- Map<String, Object> theseMatches;
- try {
- LOG.debug("Looking through {}", firstLog);
- theseMatches = substringSearch(firstLog, search, numMatches - matchCount, offset);
- } catch (InvalidRequestException e) {
- LOG.error("Can't search past end of file.", e);
- theseMatches = new HashMap<>();
- }
-
- String fileName = WorkerLogs.getTopologyPortWorkerLog(firstLog);
-
- List<Map<String, Object>> newMatches = new ArrayList<>(matches);
- Map<String, Object> currentFileMatch = new HashMap<>(theseMatches);
- currentFileMatch.put("fileName", fileName);
- List<String> splitPath;
- try {
- splitPath = Arrays.asList(firstLog.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- currentFileMatch.put("port", first(takeLast(splitPath, 2)));
- newMatches.add(currentFileMatch);
-
- int newCount = matchCount + ((List<?>)theseMatches.get("matches")).size();
-
- if (theseMatches.isEmpty()) {
- // matches and matchCount is not changed
- logs = rest(logs);
- offset = 0;
- fileOffset = fileOffset + 1;
- } else if (newCount >= numMatches) {
- matches = newMatches;
- break;
- } else {
- matches = newMatches;
- logs = rest(logs);
- offset = 0;
- fileOffset = fileOffset + 1;
- matchCount = newCount;
- }
- }
-
- return new Matched(fileOffset, search, matches);
- }
-
-
- /**
- * As the file is read into a buffer, 1/2 the buffer's size at a time, we search the buffer for matches of the
- * substring and return a list of zero or more matches.
- */
- private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, File file, int fileLength, int offsetToBuf,
- int initBufOffset, BufferedInputStream stream, Integer bytesSkipped,
- int bytesRead, ByteBuffer haystack, byte[] needle,
- List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes) throws IOException {
- int bufOffset = initBufOffset;
- List<Map<String, Object>> matches = initialMatches;
-
- byte[] newBeforeBytes;
- Integer newByteOffset;
-
- while (true) {
- int offset = offsetOfBytes(haystack.array(), needle, bufOffset);
- if (matches.size() < numMatches && offset >= 0) {
- int fileOffset = offsetToBuf + offset;
- int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length;
-
- byte[] beforeArg = null;
- byte[] afterArg = null;
- if (offset < GREP_CONTEXT_SIZE) {
- beforeArg = beforeBytes;
- }
-
- if (offset > bytesNeededAfterMatch) {
- afterArg = tryReadAhead(stream, haystack, offset, fileLength, bytesRead);
- }
-
- bufOffset = offset + needle.length;
- matches.add(mkMatchData(needle, haystack, offset, fileOffset,
- file.getCanonicalPath(), isDaemon, beforeArg, afterArg));
- } else {
- int beforeStrToOffset = Math.min(haystack.limit(), GREP_MAX_SEARCH_SIZE);
- int beforeStrFromOffset = Math.max(0, beforeStrToOffset - GREP_CONTEXT_SIZE);
- newBeforeBytes = Arrays.copyOfRange(haystack.array(), beforeStrFromOffset, beforeStrToOffset);
-
- // It's OK if new-byte-offset is negative.
- // This is normal if we are out of bytes to read from a small file.
- if (matches.size() >= numMatches) {
- newByteOffset = ((Number) last(matches).get("byteOffset")).intValue() + needle.length;
- } else {
- newByteOffset = bytesSkipped + bytesRead - GREP_MAX_SEARCH_SIZE;
- }
-
- break;
- }
- }
-
- return new SubstringSearchResult(matches, newByteOffset, newBeforeBytes);
- }
-
- private int rotateGrepBuffer(ByteBuffer buf, BufferedInputStream stream, int totalBytesRead, File file,
- int fileLength) throws IOException {
- byte[] bufArray = buf.array();
-
- // Copy the 2nd half of the buffer to the first half.
- System.arraycopy(bufArray, GREP_MAX_SEARCH_SIZE, bufArray, 0, GREP_MAX_SEARCH_SIZE);
-
- // Zero-out the 2nd half to prevent accidental matches.
- Arrays.fill(bufArray, GREP_MAX_SEARCH_SIZE, bufArray.length, (byte) 0);
-
- // Fill the 2nd half with new bytes from the stream.
- int bytesRead = stream.read(bufArray, GREP_MAX_SEARCH_SIZE, Math.min((int) fileLength, GREP_MAX_SEARCH_SIZE));
- buf.limit(GREP_MAX_SEARCH_SIZE + bytesRead);
- return totalBytesRead + bytesRead;
- }
-
-
- private Map<String, Object> mkMatchData(byte[] needle, ByteBuffer haystack, int haystackOffset, int fileOffset, String fname,
- boolean isDaemon, byte[] beforeBytes, byte[] afterBytes)
- throws UnsupportedEncodingException, UnknownHostException {
- String url;
- if (isDaemon) {
- url = urlToMatchCenteredInLogPageDaemonFile(needle, fname, fileOffset, logviewerPort);
- } else {
- url = urlToMatchCenteredInLogPage(needle, fname, fileOffset, logviewerPort);
- }
-
- byte[] haystackBytes = haystack.array();
- String beforeString;
- String afterString;
-
- if (haystackOffset >= GREP_CONTEXT_SIZE) {
- StringBuilder sb = new StringBuilder();
- sb.append(new String(haystackBytes, (haystackOffset - GREP_CONTEXT_SIZE), GREP_CONTEXT_SIZE, "UTF-8"));
- sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
- beforeString = sb.toString();
- } else {
- int numDesired = Math.max(0, GREP_CONTEXT_SIZE - haystackOffset);
- int beforeSize = beforeBytes != null ? beforeBytes.length : 0;
- int numExpected = Math.min(beforeSize, numDesired);
-
- if (numExpected > 0) {
- StringBuilder sb = new StringBuilder();
- sb.append(new String(beforeBytes, numExpected - beforeSize, numExpected, "UTF-8"));
- sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
- beforeString = sb.toString();
- } else {
- beforeString = new String(haystackBytes, 0, haystackOffset, "UTF-8");
- }
- }
-
- int needleSize = needle.length;
- int afterOffset = haystackOffset + needleSize;
- int haystackSize = haystack.limit();
-
- if ((afterOffset + GREP_CONTEXT_SIZE) < haystackSize) {
- afterString = new String(haystackBytes, afterOffset, GREP_CONTEXT_SIZE, "UTF-8");
- } else {
- int numDesired = GREP_CONTEXT_SIZE - (haystackSize - afterOffset);
- int afterSize = afterBytes != null ? afterBytes.length : 0;
- int numExpected = Math.min(afterSize, numDesired);
-
- if (numExpected > 0) {
- StringBuilder sb = new StringBuilder();
- sb.append(new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8"));
- sb.append(new String(afterBytes, 0, numExpected, "UTF-8"));
- afterString = sb.toString();
- } else {
- afterString = new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8");
- }
- }
-
- Map<String, Object> ret = new HashMap<>();
- ret.put("byteOffset", fileOffset);
- ret.put("beforeString", beforeString);
- ret.put("afterString", afterString);
- ret.put("matchString", new String(needle, "UTF-8"));
- ret.put("logviewerURL", url);
-
- return ret;
- }
-
- /**
- * Tries once to read ahead in the stream to fill the context and
- * resets the stream to its position before the call.
- */
- private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) throws IOException {
- int numExpected = Math.min(fileLength - bytesRead, GREP_CONTEXT_SIZE);
- byte[] afterBytes = new byte[numExpected];
- stream.mark(numExpected);
- // Only try reading once.
- stream.read(afterBytes, 0, numExpected);
- stream.reset();
- return afterBytes;
- }
-
- /**
- * Searches a given byte array for a match of a sub-array of bytes.
- * Returns the offset to the byte that matches, or -1 if no match was found.
- */
- private int offsetOfBytes(byte[] buffer, byte[] search, int initOffset) {
- if (search.length <= 0) {
- throw new IllegalArgumentException("Search array should not be empty.");
- }
-
- if (initOffset < 0) {
- throw new IllegalArgumentException("Start offset shouldn't be negative.");
- }
-
- int offset = initOffset;
- int candidateOffset = initOffset;
- int valOffset = 0;
- int retOffset = 0;
-
- while (true) {
- if (search.length - valOffset <= 0) {
- // found
- retOffset = candidateOffset;
- break;
- } else {
- if (offset >= buffer.length) {
- // We ran out of buffer for the search.
- retOffset = -1;
- break;
- } else {
- if (search[valOffset] != buffer[offset]) {
- // The match at this candidate offset failed, so start over with the
- // next candidate byte from the buffer.
- int newOffset = candidateOffset + 1;
-
- offset = newOffset;
- candidateOffset = newOffset;
- valOffset = 0;
- } else {
- // So far it matches. Keep going...
- offset = offset + 1;
- valOffset = valOffset + 1;
- }
- }
- }
- }
-
- return retOffset;
- }
-
- /**
- * This response data only includes a next byte offset if there is more of the file to read.
- */
- private Map<String, Object> mkGrepResponse(byte[] searchBytes, Integer offset, List<Map<String, Object>> matches,
- Integer nextByteOffset) throws UnsupportedEncodingException {
- Map<String, Object> ret = new HashMap<>();
- ret.put("searchString", new String(searchBytes, "UTF-8"));
- ret.put("startByteOffset", offset);
- ret.put("matches", matches);
- if (nextByteOffset != null) {
- ret.put("nextByteOffset", nextByteOffset);
- }
- return ret;
- }
-
- private String urlToMatchCenteredInLogPage(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
- String host = Utils.hostname();
- String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
- takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 3));
-
- Map<String, Object> parameters = new HashMap<>();
- parameters.put("file", splittedFileName);
- parameters.put("start", Math.max(0, offset - (DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
- parameters.put("length", DEFAULT_BYTES_PER_PAGE);
-
- return URLBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
- }
-
- private String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
- String host = Utils.hostname();
- String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
- takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 1));
-
- Map<String, Object> parameters = new HashMap<>();
- parameters.put("file", splittedFileName);
- parameters.put("start", Math.max(0, offset - (DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
- parameters.put("length", DEFAULT_BYTES_PER_PAGE);
-
- return URLBuilder.build(String.format("http://%s:%d/api/v1/daemonlog", host, port), parameters);
- }
-
- private static class Matched implements JSONAware {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private int fileOffset;
- private String searchString;
- private List<Map<String, Object>> matches;
-
- public Matched(int fileOffset, String searchString, List<Map<String, Object>> matches) {
- this.fileOffset = fileOffset;
- this.searchString = searchString;
- this.matches = matches;
- }
-
- public int getFileOffset() {
- return fileOffset;
- }
-
- public String getSearchString() {
- return searchString;
- }
-
- public List<Map<String, Object>> getMatches() {
- return matches;
- }
-
- @Override
- public String toJSONString() {
- try {
- return OBJECT_MAPPER.writeValueAsString(this);
- } catch (JsonProcessingException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private static class SubstringSearchResult {
- private List<Map<String, Object>> matches;
- private Integer newByteOffset;
- private byte[] newBeforeBytes;
-
- public SubstringSearchResult(List<Map<String, Object>> matches, Integer newByteOffset, byte[] newBeforeBytes) {
- this.matches = matches;
- this.newByteOffset = newByteOffset;
- this.newBeforeBytes = newBeforeBytes;
- }
-
- public List<Map<String, Object>> getMatches() {
- return matches;
- }
-
- public Integer getNewByteOffset() {
- return newByteOffset;
- }
-
- public byte[] getNewBeforeBytes() {
- return newBeforeBytes;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java
deleted file mode 100644
index 1cc274c..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.handler;
-
-import j2html.tags.DomContent;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.daemon.DirectoryCleaner;
-import org.apache.storm.daemon.wip.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.utils.ServerUtils;
-
-import javax.ws.rs.core.Response;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static j2html.TagCreator.a;
-import static j2html.TagCreator.body;
-import static j2html.TagCreator.head;
-import static j2html.TagCreator.html;
-import static j2html.TagCreator.li;
-import static j2html.TagCreator.link;
-import static j2html.TagCreator.title;
-import static j2html.TagCreator.ul;
-import static java.util.stream.Collectors.toList;
-
-public class LogviewerProfileHandler {
-
- public static final String WORKER_LOG_FILENAME = "worker.log";
- private final String logRoot;
- private final ResourceAuthorizer resourceAuthorizer;
-
- public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer) {
- this.logRoot = logRoot;
- this.resourceAuthorizer = resourceAuthorizer;
- }
-
- public Response listDumpFiles(String topologyId, String hostPort, String user) throws IOException {
- String portStr = hostPort.split(":")[1];
- File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
-
- if (dir.exists()) {
- String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
- if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
- String content = buildDumpFileListPage(topologyId, hostPort, dir);
- return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
- } else {
- return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
- }
- } else {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- }
- }
-
- private String buildDumpFileListPage(String topologyId, String hostPort, File dir) throws IOException {
- List<DomContent> liTags = getProfilerDumpFiles(dir).stream()
- .map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + file)))
- .collect(toList());
-
- return html(
- head(
- title("File Dumps - Storm Log Viewer"),
- link().withRel("stylesheet").withHref("/css/bootstrap-3.3.1.min.css"),
- link().withRel("stylesheet").withHref("/css/jquery.dataTables.1.10.4.min.css"),
- link().withRel("stylesheet").withHref("/css/style.css")
- ),
- body(
- ul(liTags.toArray(new DomContent[]{}))
- )
- ).render();
- }
-
- public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException {
- String portStr = hostPort.split(":")[1];
- File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
- File file = new File(dir, fileName);
-
- if (dir.exists() && file.exists()) {
- String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
- if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
- return LogviewerResponseBuilder.buildDownloadFile(file);
- } else {
- return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
- }
- } else {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- }
- }
-
- private List<String> getProfilerDumpFiles(File dir) throws IOException {
- List<File> filesForDir = DirectoryCleaner.getFilesForDir(dir);
- return filesForDir.stream().filter(file -> {
- String fileName = file.getName();
- return StringUtils.isNotEmpty(fileName)
- && (fileName.endsWith(".txt") || fileName.endsWith(".jfr") || fileName.endsWith(".bin"));
- }).map(File::getName).collect(toList());
- }
-
-}
[19/19] storm git commit: Added STORM-1280 to Changelog
Posted by bo...@apache.org.
Added STORM-1280 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3c53dd51
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3c53dd51
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3c53dd51
Branch: refs/heads/master
Commit: 3c53dd516810824593855060a69c5e82d08323e7
Parents: ad5ea39
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Mon Jul 24 10:48:02 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Mon Jul 24 10:48:02 2017 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3c53dd51/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68b51e9..0f9eea0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-1280: port backtype.storm.daemon.logviewer to java
* STORM-2644: Show message when result not found on deep search page
* STORM-2643: Add a confirmation dialog for profiling and debugging actions on Storm UI
* STORM-2641: Make storm.py print output from subprocess on error on Windows, make storm.ps1 return the right exit code
[18/19] storm git commit: Merge branch 'STORM-1280' of
https://github.com/HeartSaVioR/storm into STORM-1280
Posted by bo...@apache.org.
Merge branch 'STORM-1280' of https://github.com/HeartSaVioR/storm into STORM-1280
STORM-1280: port backtype.storm.daemon.logviewer to java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ad5ea39f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ad5ea39f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ad5ea39f
Branch: refs/heads/master
Commit: ad5ea39facc10dc2b9d07c4d12c0be0208dca00e
Parents: 1622f19 8b4a1ce
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Mon Jul 24 10:47:41 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Mon Jul 24 10:47:41 2017 -0500
----------------------------------------------------------------------
bin/storm.py | 8 +-
.../org/apache/storm/st/wrapper/TopoWrap.java | 4 +-
pom.xml | 9 +
storm-buildtools/storm_checkstyle.xml | 6 +
.../org/apache/storm/daemon/StormCommon.java | 2 -
.../clj/org/apache/storm/daemon/logviewer.clj | 1254 ------------------
storm-core/src/clj/org/apache/storm/ui/core.clj | 10 +-
.../dev/logviewer-search-context-tests.log.gz | Bin 72 -> 0 bytes
.../dev/logviewer-search-context-tests.log.test | 1 -
storm-core/src/dev/small-worker.log.test | 1 -
storm-core/src/dev/test-3072.log.test | 3 -
storm-core/src/dev/test-worker.log.test | 380 ------
.../apache/storm/daemon/DirectoryCleaner.java | 174 ---
storm-core/src/ui/public/component.html | 2 +-
.../src/ui/public/deep_search_result.html | 2 +-
storm-core/src/ui/public/logviewer_search.html | 2 +-
storm-core/src/ui/public/search_result.html | 4 +-
.../clj/org/apache/storm/logviewer_test.clj | 824 ------------
.../storm/hack/StormShadeTransformer.java | 5 -
.../reporters/JmxPreparableReporter.java | 1 -
.../storm/daemon/supervisor/BasicContainer.java | 1 -
.../apache/storm/utils/ServerConfigUtils.java | 3 +-
storm-webapp/pom.xml | 51 +-
.../common/AuthorizationExceptionMapper.java | 40 +
.../daemon/common/JsonResponseBuilder.java | 74 ++
.../webapp/AuthorizationExceptionMapper.java | 40 -
.../daemon/drpc/webapp/DRPCApplication.java | 1 +
.../daemon/logviewer/LogviewerConstant.java | 29 +
.../storm/daemon/logviewer/LogviewerServer.java | 172 +++
.../handler/LogviewerLogDownloadHandler.java | 71 +
.../handler/LogviewerLogPageHandler.java | 470 +++++++
.../handler/LogviewerLogSearchHandler.java | 753 +++++++++++
.../handler/LogviewerProfileHandler.java | 140 ++
.../logviewer/utils/DirectoryCleaner.java | 196 +++
.../daemon/logviewer/utils/LogCleaner.java | 267 ++++
.../logviewer/utils/LogFileDownloader.java | 67 +
.../utils/LogviewerResponseBuilder.java | 157 +++
.../logviewer/utils/ResourceAuthorizer.java | 154 +++
.../daemon/logviewer/utils/WorkerLogs.java | 219 +++
.../logviewer/webapp/LogviewerApplication.java | 102 ++
.../logviewer/webapp/LogviewerResource.java | 269 ++++
.../daemon/utils/ListFunctionalSupport.java | 108 ++
.../apache/storm/daemon/utils/StreamUtil.java | 47 +
.../apache/storm/daemon/utils/UrlBuilder.java | 53 +
.../storm/daemon/drpc/DRPCServerTest.java | 13 +-
.../handler/LogviewerLogPageHandlerTest.java | 105 ++
.../handler/LogviewerLogSearchHandlerTest.java | 856 ++++++++++++
.../testsupport/ArgumentsVerifier.java | 43 +
.../testsupport/MockDirectoryBuilder.java | 72 +
.../logviewer/testsupport/MockFileBuilder.java | 72 +
.../daemon/logviewer/utils/LogCleanerTest.java | 350 +++++
.../logviewer/utils/ResourceAuthorizerTest.java | 182 +++
.../daemon/logviewer/utils/WorkerLogsTest.java | 75 ++
.../logviewer-search-context-tests.log.gz | Bin 0 -> 72 bytes
.../logviewer-search-context-tests.log.test | 1 +
.../src/test/resources/small-worker.log.test | 1 +
.../src/test/resources/test-3072.log.test | 3 +
.../src/test/resources/test-worker.log.test | 380 ++++++
58 files changed, 5609 insertions(+), 2720 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad5ea39f/bin/storm.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad5ea39f/storm-core/src/clj/org/apache/storm/ui/core.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad5ea39f/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad5ea39f/storm-core/src/ui/public/deep_search_result.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad5ea39f/storm-core/src/ui/public/logviewer_search.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad5ea39f/storm-core/src/ui/public/search_result.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad5ea39f/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
[02/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/resources/test-worker.log.test
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/resources/test-worker.log.test b/storm-webapp/src/test/resources/test-worker.log.test
new file mode 100644
index 0000000..8fb4c53
--- /dev/null
+++ b/storm-webapp/src/test/resources/test-worker.log.test
@@ -0,0 +1,380 @@
+Test needle is near the beginning of the file.
+This file assumes a buffer size of 2048 bytes, a max search string size of 1024 bytes, and a context length of 128 UTF-8 characters.
+The early match tests the case when we find a match too close to the start of the file to give the normal before context strings.
+
+padding 5
+padding 6
+padding 7
+padding 8
+padding 9
+padding 10
+padding 11
+padding 12
+padding 13
+padding 14
+padding 15
+padding 16
+padding 17
+padding 18
+padding 19
+padding 20
+padding 21
+padding 22
+padding 23
+padding 24
+padding 25
+padding 26
+padding 27
+padding 28
+padding 29
+padding 30
+padding 31
+padding 32
+padding 33
+padding 34
+padding 35
+padding 36
+padding 37
+padding 38
+padding 39
+padding 40
+padding 41
+padding 42
+padding 43
+padding 44
+padding 45
+padding 46
+padding 47
+padding 48
+padding 49
+padding 50
+padding 51
+padding 52
+padding 53
+padding 54
+padding 55
+padding 56
+padding 57
+padding 58
+padding 59
+padding 60
+padding 61
+padding 62
+padding 63
+padding 64
+padding 65
+padding 66
+padding 67
+padding 68
+padding 69
+padding 70
+padding 71
+padding 72
+padding 73
+padding 74
+padding 75
+padding 76
+padding 77
+padding 78
+padding 79
+padding 80
+padding 81
+padding 82
+padding 83
+padding 84
+padding 85
+padding 86
+padding 87
+padding 88
+padding 89
+padding 90
+padding 91
+padding 92
+padding 93
+padding 94
+padding 95
+padding 96
+padding 97
+padding 98
+padding 99
+padding 100
+padding 101
+padding 102
+padding 103
+padding 104
+padding 105
+padding 106
+padding 107
+padding 108
+padding 109
+padding 110
+padding 111
+padding 112
+padding 113
+padding 114
+padding 115
+padding 116
+padding 117
+padding 118
+padding 119
+padding 120
+padding 121
+padding 122
+padding 123
+padding 124
+padding 125
+padding 126
+padding 127
+padding 128
+padding 129
+padding 130
+padding 131
+padding 132
+padding 133
+padding 134
+padding 135
+padding 136
+padding 137
+padding 138
+padding 139
+padding 140
+padding 141
+padding 142
+padding 143
+padding 144
+padding 145
+padding 146
+padding 147
+padding 148
+padding 149
+padding 150
+padding 151
+padding 152
+padding 153
+Near the end of a 1024 byte block, a needle.
+A needle that straddles a 1024 byte boundary should also be detected.
+
+padding 157
+padding 158
+padding 159
+padding 160
+padding 161
+padding 162
+padding 163
+padding 164
+padding 165
+padding 166
+padding 167
+padding 168
+padding 169
+padding 170
+padding 171
+padding 172
+padding 173
+padding 174
+padding 175
+padding 176
+padding 177
+padding 178
+padding 179
+padding 180
+padding 181
+padding 182
+padding 183
+padding 184
+padding 185
+padding 186
+padding 187
+padding 188
+padding 189
+padding 190
+padding 191
+padding 192
+padding 193
+padding 194
+padding 195
+padding 196
+padding 197
+padding 198
+padding 199
+padding 200
+padding 201
+padding 202
+padding 203
+padding 204
+padding 205
+padding 206
+padding 207
+padding 208
+padding 209
+padding 210
+padding 211
+padding 212
+padding 213
+padding 214
+padding 215
+padding 216
+padding 217
+padding 218
+padding 219
+padding 220
+padding 221
+padding 222
+padding 223
+padding 224
+padding 225
+padding 226
+padding 227
+padding 228
+padding 229
+padding 230
+padding 231
+padding 232
+padding 233
+padding 234
+padding 235
+
+
+Here a needle occurs just after a 1024 byte boundary. It should have the correct context.
+
+Text with two adjoining matches: needleneedle
+
+padding 243
+padding 244
+padding 245
+padding 246
+padding 247
+padding 248
+padding 249
+padding 250
+padding 251
+padding 252
+padding 253
+padding 254
+padding 255
+padding 256
+padding 257
+padding 258
+padding 259
+padding 260
+padding 261
+padding 262
+padding 263
+padding 264
+padding 265
+padding 266
+padding 267
+padding 268
+padding 269
+padding 270
+padding 271
+padding 272
+padding 273
+padding 274
+padding 275
+padding 276
+padding 277
+padding 278
+padding 279
+padding 280
+padding 281
+padding 282
+padding 283
+padding 284
+padding 285
+padding 286
+padding 287
+padding 288
+padding 289
+padding 290
+padding 291
+padding 292
+padding 293
+padding 294
+padding 295
+padding 296
+padding 297
+padding 298
+padding 299
+padding 300
+padding 301
+padding 302
+padding 303
+padding 304
+
+The following match of 1024 bytes completely fills half the byte buffer. It is a search substring of the maximum size......
+
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
XXXXXXXXXXXXXXXXXXXXXXXXXXX
+The following max-size match straddles a 1024 byte buffer.
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
XXXXXXXXXXXXXXXXXXXXXXXXXXX
+
+Here are four non-ascii 1-byte UTF-8 characters: αβγδε
+
+needle
+
+Here are four printable 2-byte UTF-8 characters: ¡¢£¤¥
+
+needle
+
+
+
+Here are four printable 3-byte UTF-8 characters: ऄअआइई
+
+needle
+
+Here are four printable 4-byte UTF-8 characters: 𐄇𐄈𐄉𐄋𐄌
+
+needle
+
+Here are four of the same invalid UTF-8 characters: ����������������
+
+needle
+
+padding 332
+padding 333
+padding 334
+padding 335
+padding 336
+padding 337
+padding 338
+padding 339
+padding 340
+padding 341
+padding 342
+padding 343
+padding 344
+padding 345
+padding 346
+padding 347
+padding 348
+padding 349
+padding 350
+padding 351
+padding 352
+padding 353
+padding 354
+padding 355
+padding 356
+padding 357
+padding 358
+padding 359
+padding 360
+padding 361
+padding 362
+padding 363
+padding 364
+padding 365
+padding 366
+padding 367
+padding 368
+padding 369
+padding 370
+padding 371
+padding 372
+padding 373
+padding 374
+padding 375
+
+The following tests multibyte UTF-8 Characters straddling the byte boundary: 𐄀𐄁𐄂
+
+needle
\ No newline at end of file
[16/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* fix a bug: parameter swapped
* move 'file' param for '/search' from path to query param
* also applied to static html files
* select current file in file list dropdown in log page
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/35ca265a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/35ca265a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/35ca265a
Branch: refs/heads/master
Commit: 35ca265af963dc21b356e1a305fea897c58a59fa
Parents: 37f9c59
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Jul 24 11:44:28 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Jul 24 11:44:28 2017 +0900
----------------------------------------------------------------------
storm-core/src/ui/public/logviewer_search.html | 2 +-
storm-core/src/ui/public/search_result.html | 2 +-
.../org/apache/storm/utils/ServerConfigUtils.java | 3 ++-
.../logviewer/handler/LogviewerLogPageHandler.java | 17 ++++++++++-------
.../handler/LogviewerLogSearchHandler.java | 2 +-
.../daemon/logviewer/webapp/LogviewerResource.java | 5 +++--
6 files changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/35ca265a/storm-core/src/ui/public/logviewer_search.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/logviewer_search.html b/storm-core/src/ui/public/logviewer_search.html
index 89a7854..96ecd8c 100644
--- a/storm-core/src/ui/public/logviewer_search.html
+++ b/storm-core/src/ui/public/logviewer_search.html
@@ -55,7 +55,7 @@ $(document).ready(function() {
$("#search-form").append(Mustache.render($(template).filter("#search-single-file").html(),{file: file, search: search, isDaemon: isDaemon}));
var result = $("#result");
- var url = "/api/v1/search/"+encodeURIComponent(file)+"?search-string="+search+"&start-byte-offset="+offset+"&is-daemon="+isDaemon;
+ var url = "/api/v1/search?file="+encodeURIComponent(file)+"&search-string="+search+"&start-byte-offset="+offset+"&is-daemon="+isDaemon;
$.getJSON(url,function(response,status,jqXHR) {
response.file = file;
result.append(Mustache.render($(template).filter("#logviewer-search-result-template").html(),response));
http://git-wip-us.apache.org/repos/asf/storm/blob/35ca265a/storm-core/src/ui/public/search_result.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/search_result.html b/storm-core/src/ui/public/search_result.html
index 216a700..d6ebb44 100644
--- a/storm-core/src/ui/public/search_result.html
+++ b/storm-core/src/ui/public/search_result.html
@@ -68,7 +68,7 @@ $(document).ready(function() {
var port = response.hostPortList[index].port;
var elemId = response.hostPortList[index].elemId;
var file = id+"/"+port+"/worker.log";
- var searchURL = "http://"+host+":"+logviewerPort+"/api/v1/search/"+encodeURIComponent(file)+"?search-string="+search+"&num-matches="+count;
+ var searchURL = "http://"+host+":"+logviewerPort+"/api/v1/search?file="+encodeURIComponent(file)+"&search-string="+search+"&num-matches="+count;
if (searchArchived != "") {
searchURL = "http://"+host+":"+logviewerPort+"/api/v1/deepSearch/"+id+"?search-string="+search+"&num-matches="+count+"&search-archived=true&port="+port;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/35ca265a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
index cd63395..426e190 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
@@ -30,6 +30,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Pattern;
public class ServerConfigUtils {
public static final String FILE_SEPARATOR = File.separator;
@@ -161,7 +162,7 @@ public class ServerConfigUtils {
}
public static File getLogMetaDataFile(String fname) {
- String[] subStrings = fname.split(FILE_SEPARATOR); // TODO: does this work well on windows?
+ String[] subStrings = fname.split(Pattern.quote(FILE_SEPARATOR)); // TODO: does this work well on windows?
String id = subStrings[0];
Integer port = Integer.parseInt(subStrings[1]);
return getLogMetaDataFile(Utils.readStormConfig(), id, port);
http://git-wip-us.apache.org/repos/asf/storm/blob/35ca265a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
index 47fa4b3..c6e9828 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
@@ -27,6 +27,7 @@ import static j2html.TagCreator.head;
import static j2html.TagCreator.html;
import static j2html.TagCreator.input;
import static j2html.TagCreator.link;
+import static j2html.TagCreator.option;
import static j2html.TagCreator.p;
import static j2html.TagCreator.pre;
import static j2html.TagCreator.select;
@@ -226,7 +227,7 @@ public class LogviewerLogPageHandler {
bodyContents.add(searchFileForm(fileName, "no"));
// list all files for this topology
- bodyContents.add(logFileSelectionForm(reorderedFilesStr, "log"));
+ bodyContents.add(logFileSelectionForm(reorderedFilesStr, fileName, "log"));
if (pagerData != null) {
bodyContents.add(pagerData);
}
@@ -306,7 +307,7 @@ public class LogviewerLogPageHandler {
bodyContents.add(searchFileForm(fileName, "yes"));
// list all daemon logs
- bodyContents.add(logFileSelectionForm(reorderedFilesStr, "daemonlog"));
+ bodyContents.add(logFileSelectionForm(reorderedFilesStr, fileName, "daemonlog"));
if (pagerData != null) {
bodyContents.add(pagerData);
}
@@ -360,16 +361,18 @@ public class LogviewerLogPageHandler {
return a(content).withHref(url);
}
- private DomContent logFileSelectionForm(List<String> logFiles, String type) {
+ private DomContent logFileSelectionForm(List<String> logFiles, String selectedFile, String type) {
return form(
- dropDown("file", logFiles),
+ dropDown("file", logFiles, selectedFile),
input().withType("submit").withValue("Switch file")
).withAction(type).withId("list-of-files");
}
- private DomContent dropDown(String name, List<String> logFiles) {
- List<DomContent> options = logFiles.stream().map(TagCreator::option).collect(toList());
- return select(options.toArray(new DomContent[]{})).withName(name).withId(name);
+ private DomContent dropDown(String name, List<String> logFiles, String selectedFile) {
+ List<DomContent> options = logFiles.stream()
+ .map(file -> option(file).condAttr(file.equals(selectedFile), "selected", "selected"))
+ .collect(toList());
+ return select(options.toArray(new DomContent[]{})).withName(name).withId(name).withValue(selectedFile);
}
private DomContent searchFileForm(String fileName, String isDaemonValue) {
http://git-wip-us.apache.org/repos/asf/storm/blob/35ca265a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
index 76f1683..117cb86 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
@@ -119,7 +119,7 @@ public class LogviewerLogSearchHandler {
File file = new File(rootDir, fileName).getCanonicalFile();
Response response;
if (file.exists()) {
- if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
+ if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
Integer numMatchesInt = numMatchesStr != null ? tryParseIntParam("num-matches", numMatchesStr) : null;
Integer offsetInt = offsetStr != null ? tryParseIntParam("start-byte-offset", offsetStr) : null;
http://git-wip-us.apache.org/repos/asf/storm/blob/35ca265a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
index 478001d..6478ca6 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
@@ -214,10 +214,11 @@ public class LogviewerResource {
* Handles '/search' (searching from specific worker or daemon log file) request.
*/
@GET
- @Path("/search/{file}")
- public Response search(@PathParam("file") String file, @Context HttpServletRequest request) throws IOException {
+ @Path("/search")
+ public Response search(@Context HttpServletRequest request) throws IOException {
String user = httpCredsHandler.getUserName(request);
boolean isDaemon = StringUtils.equals(request.getParameter("is-daemon"), "yes");
+ String file = request.getParameter("file");
String decodedFileName = URLDecoder.decode(file);
String searchString = request.getParameter("search-string");
String numMatchesStr = request.getParameter("num-matches");
[06/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
new file mode 100644
index 0000000..7d800e7
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
@@ -0,0 +1,707 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.handler;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.daemon.logviewer.LogviewerConstant;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.URLBuilder;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.first;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.last;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+
+public class LogviewerLogSearchHandler {
+ private final static Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class);
+
+ public static final int GREP_MAX_SEARCH_SIZE = 1024;
+ public static final int GREP_BUF_SIZE = 2048;
+ public static final int GREP_CONTEXT_SIZE = 128;
+ public static final Pattern WORKER_LOG_FILENAME_PATTERN = Pattern.compile("^worker.log(.*)");
+
+ private final Map<String, Object> stormConf;
+ private final String logRoot;
+ private final String daemonLogRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+ private final Integer logviewerPort;
+
+ public LogviewerLogSearchHandler(Map<String, Object> stormConf, String logRoot, String daemonLogRoot,
+ ResourceAuthorizer resourceAuthorizer) {
+ this.stormConf = stormConf;
+ this.logRoot = logRoot;
+ this.daemonLogRoot = daemonLogRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+
+ this.logviewerPort = ObjectReader.getInt(stormConf.get(DaemonConfig.LOGVIEWER_PORT));
+ }
+
+ public Response searchLogFile(String fileName, String user, boolean isDaemon, String search,
+ String numMatchesStr, String offsetStr, String callback, String origin)
+ throws IOException, InvalidRequestException {
+ String rootDir = isDaemon ? daemonLogRoot : logRoot;
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ Response response;
+ if (file.exists()) {
+ if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
+ Integer numMatchesInt = numMatchesStr != null ? tryParseIntParam("num-matches", numMatchesStr) : null;
+ Integer offsetInt = offsetStr != null ? tryParseIntParam("start-byte-offset", offsetStr) : null;
+
+ try {
+ if (StringUtils.isNotEmpty(search) && search.getBytes("UTF-8").length <= GREP_MAX_SEARCH_SIZE) {
+ Map<String, Object> entity = new HashMap<>();
+ entity.put("isDaemon", isDaemon ? "yes" : "no");
+ entity.putAll(substringSearch(file, search, isDaemon, numMatchesInt, offsetInt));
+
+ response = LogviewerResponseBuilder.buildSuccessJsonResponse(entity, callback, origin);
+ } else {
+ throw new InvalidRequestException("Search substring must be between 1 and 1024 "
+ + "UTF-8 bytes in size (inclusive)");
+ }
+ } catch (Exception ex) {
+ response = LogviewerResponseBuilder.buildExceptionJsonResponse(ex, callback);
+ }
+ } else {
+ // unauthorized
+ response = LogviewerResponseBuilder.buildUnauthorizedUserJsonResponse(user, callback);
+ }
+ } else {
+ // not found
+ Map<String, String> entity = new HashMap<>();
+ entity.put("error", "Not Found");
+ entity.put("errorMessage", "The file was not found on this node.");
+
+ response = new JsonResponseBuilder().setData(entity).setCallback(callback).setStatus(404).build();
+ }
+
+ return response;
+ }
+
+ public Response deepSearchLogsForTopology(String topologyId, String user, String search,
+ String numMatchesStr, String portStr, String fileOffsetStr, String offsetStr,
+ Boolean searchArchived, String callback, String origin) {
+ String rootDir = logRoot;
+ Object returnValue;
+ File topologyDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId);
+ if (StringUtils.isEmpty(search) || !topologyDir.exists()) {
+ returnValue = new ArrayList<>();
+ } else {
+ int fileOffset = ObjectReader.getInt(fileOffsetStr, 0);
+ int offset = ObjectReader.getInt(offsetStr, 0);
+ int numMatches = ObjectReader.getInt(numMatchesStr, 1);
+
+ File[] portDirsArray = topologyDir.listFiles();
+ List<File> portDirs;
+ if (portDirsArray != null) {
+ portDirs = Arrays.asList(portDirsArray);
+ } else {
+ portDirs = new ArrayList<>();
+ }
+
+ if (StringUtils.isEmpty(portStr) || portStr.equals("*")) {
+ // check for all ports
+ List<List<File>> filteredLogs = portDirs.stream()
+ .map(portDir -> logsForPort(user, portDir))
+ .filter(logs -> logs != null && !logs.isEmpty())
+ .collect(toList());
+
+ if (BooleanUtils.isTrue(searchArchived)) {
+ returnValue = filteredLogs.stream()
+ .map(fl -> findNMatches(fl, numMatches, 0, 0, search))
+ .collect(toList());
+ } else {
+ returnValue = filteredLogs.stream()
+ .map(fl -> Collections.singletonList(first(fl)))
+ .map(fl -> findNMatches(fl, numMatches, 0, 0, search))
+ .collect(toList());
+ }
+ } else {
+ int port = Integer.parseInt(portStr);
+ // check just the one port
+ List<Integer> slotsPorts = (List<Integer>) stormConf.getOrDefault(DaemonConfig.SUPERVISOR_SLOTS_PORTS,
+ new ArrayList<>());
+ boolean containsPort = slotsPorts.stream()
+ .anyMatch(slotPort -> slotPort != null && (slotPort == port));
+ if (!containsPort) {
+ returnValue = new ArrayList<>();
+ } else {
+ File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId +
+ Utils.FILE_PATH_SEPARATOR + port);
+
+ if (!portDir.exists() || logsForPort(user, portDir).isEmpty()) {
+ returnValue = new ArrayList<>();
+ } else {
+ List<File> filteredLogs = logsForPort(user, portDir);
+ if (BooleanUtils.isTrue(searchArchived)) {
+ returnValue = findNMatches(filteredLogs, numMatches, fileOffset, offset, search);
+ } else {
+ returnValue = findNMatches(Collections.singletonList(first(filteredLogs)),
+ numMatches, 0, offset, search);
+ }
+ }
+ }
+ }
+ }
+
+ return LogviewerResponseBuilder.buildSuccessJsonResponse(returnValue, callback, origin);
+ }
+
+ private Integer tryParseIntParam(String paramName, String value) throws InvalidRequestException {
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new InvalidRequestException("Could not parse " + paramName + " to an integer");
+ }
+ }
+
+ @VisibleForTesting
+ Map<String,Object> substringSearch(File file, String searchString) throws InvalidRequestException {
+ return substringSearch(file, searchString, false, 10, 0);
+ }
+
+ @VisibleForTesting
+ Map<String,Object> substringSearch(File file, String searchString, int numMatches) throws InvalidRequestException {
+ return substringSearch(file, searchString, false, numMatches, 0);
+ }
+
+ @VisibleForTesting
+ Map<String,Object> substringSearchDaemonLog(File file, String searchString) throws InvalidRequestException {
+ return substringSearch(file, searchString, true, 10, 0);
+ }
+
+ @VisibleForTesting
+ Map<String,Object> substringSearch(File file, String searchString, int numMatches, int startByteOffset) throws InvalidRequestException {
+ return substringSearch(file, searchString, false, numMatches, startByteOffset);
+ }
+
+ private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches, Integer startByteOffset) throws InvalidRequestException {
+ try {
+ if (StringUtils.isEmpty(searchString)) {
+ throw new IllegalArgumentException("Precondition fails: search string should not be empty.");
+ }
+ if (searchString.getBytes("UTF-8").length > GREP_MAX_SEARCH_SIZE) {
+ throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + GREP_MAX_SEARCH_SIZE);
+ }
+
+ boolean isZipFile = file.getName().endsWith(".gz");
+ FileInputStream fis = new FileInputStream(file);
+ InputStream gzippedInputStream;
+ if (isZipFile) {
+ gzippedInputStream = new GZIPInputStream(fis);
+ } else {
+ gzippedInputStream = fis;
+ }
+
+ BufferedInputStream stream = new BufferedInputStream(gzippedInputStream);
+
+ int fileLength;
+ if (isZipFile) {
+ fileLength = (int) ServerUtils.zipFileSize(file);
+ } else {
+ fileLength = (int) file.length();
+ }
+
+ ByteBuffer buf = ByteBuffer.allocate(GREP_BUF_SIZE);
+ byte[] bufArray = buf.array();
+ int totalBytesRead = 0;
+ byte[] searchBytes = searchString.getBytes("UTF-8");
+ numMatches = numMatches != null ? numMatches : 10;
+ startByteOffset = startByteOffset != null ? startByteOffset : 0;
+
+ // Start at the part of the log file we are interested in.
+ // Allow searching when start-byte-offset == file-len so it doesn't blow up on 0-length files
+ if (startByteOffset > fileLength) {
+ throw new InvalidRequestException("Cannot search past the end of the file");
+ }
+
+ if (startByteOffset > 0) {
+ StreamUtil.skipBytes(stream, startByteOffset);
+ }
+
+ Arrays.fill(bufArray, (byte) 0);
+
+ int bytesRead = stream.read(bufArray, 0, Math.min((int) fileLength, GREP_BUF_SIZE));
+ buf.limit(bytesRead);
+ totalBytesRead += bytesRead;
+
+ List<Map<String, Object>> initialMatches = new ArrayList<>();
+ int initBufOffset = 0;
+ int byteOffset = startByteOffset;
+ byte[] beforeBytes = null;
+
+ Map<String, Object> ret = new HashMap<>();
+ while (true) {
+ SubstringSearchResult searchRet = bufferSubstringSearch(isDaemon, file, fileLength, byteOffset, initBufOffset,
+ stream, startByteOffset, totalBytesRead, buf, searchBytes, initialMatches, numMatches, beforeBytes);
+
+ List<Map<String, Object>> matches = searchRet.getMatches();
+ Integer newByteOffset = searchRet.getNewByteOffset();
+ byte[] newBeforeBytes = searchRet.getNewBeforeBytes();
+
+ if (matches.size() < numMatches && totalBytesRead + startByteOffset < fileLength) {
+ // The start index is positioned to find any possible
+ // occurrence search string that did not quite fit in the
+ // buffer on the previous read.
+ int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length;
+
+ totalBytesRead = rotateGrepBuffer(buf, stream, totalBytesRead, file, fileLength);
+ if (totalBytesRead < 0) {
+ throw new InvalidRequestException("Cannot search past the end of the file");
+ }
+
+ initialMatches = matches;
+ initBufOffset = newBufOffset;
+ byteOffset = newByteOffset;
+ beforeBytes = newBeforeBytes;
+ } else {
+ ret.put("isDaemon", isDaemon ? "yes" : "no");
+ Integer nextByteOffset = null;
+ if (matches.size() >= numMatches || totalBytesRead < fileLength) {
+ nextByteOffset = (Integer) last(matches).get("byteOffset") + searchBytes.length;
+ if (fileLength <= nextByteOffset) {
+ nextByteOffset = null;
+ }
+ }
+ ret.putAll(mkGrepResponse(searchBytes, startByteOffset, matches, nextByteOffset));
+ break;
+ }
+ }
+
+ return ret;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * Get the filtered, authorized, sorted log files for a port.
+ */
+ @VisibleForTesting
+ List<File> logsForPort(String user, File portDir) {
+ try {
+ List<File> workerLogs = DirectoryCleaner.getFilesForDir(portDir).stream()
+ .filter(file -> WORKER_LOG_FILENAME_PATTERN.asPredicate().test(file.getName()))
+ .collect(toList());
+
+ return workerLogs.stream()
+ .filter(log -> resourceAuthorizer.isUserAllowedToAccessFile(user, WorkerLogs.getTopologyPortWorkerLog(log)))
+ .sorted((f1, f2) -> (int) (f2.lastModified() - f1.lastModified()))
+ .collect(toList());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @VisibleForTesting
+ Matched findNMatches(List<File> logs, int numMatches, int fileOffset, int offset, String search) {
+ logs = drop(logs, fileOffset);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+ int matchCount = 0;
+
+ while (true) {
+ if (logs.isEmpty()) {
+ break;
+ }
+
+ File firstLog = logs.get(0);
+ Map<String, Object> theseMatches;
+ try {
+ LOG.debug("Looking through {}", firstLog);
+ theseMatches = substringSearch(firstLog, search, numMatches - matchCount, offset);
+ } catch (InvalidRequestException e) {
+ LOG.error("Can't search past end of file.", e);
+ theseMatches = new HashMap<>();
+ }
+
+ String fileName = WorkerLogs.getTopologyPortWorkerLog(firstLog);
+
+ List<Map<String, Object>> newMatches = new ArrayList<>(matches);
+ Map<String, Object> currentFileMatch = new HashMap<>(theseMatches);
+ currentFileMatch.put("fileName", fileName);
+ List<String> splitPath;
+ try {
+ splitPath = Arrays.asList(firstLog.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ currentFileMatch.put("port", first(takeLast(splitPath, 2)));
+ newMatches.add(currentFileMatch);
+
+ int newCount = matchCount + ((List<?>)theseMatches.get("matches")).size();
+
+ if (theseMatches.isEmpty()) {
+ // matches and matchCount is not changed
+ logs = rest(logs);
+ offset = 0;
+ fileOffset = fileOffset + 1;
+ } else if (newCount >= numMatches) {
+ matches = newMatches;
+ break;
+ } else {
+ matches = newMatches;
+ logs = rest(logs);
+ offset = 0;
+ fileOffset = fileOffset + 1;
+ matchCount = newCount;
+ }
+ }
+
+ return new Matched(fileOffset, search, matches);
+ }
+
+
+ /**
+ * As the file is read into a buffer, 1/2 the buffer's size at a time, we search the buffer for matches of the
+ * substring and return a list of zero or more matches.
+ */
+ private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, File file, int fileLength, int offsetToBuf,
+ int initBufOffset, BufferedInputStream stream, Integer bytesSkipped,
+ int bytesRead, ByteBuffer haystack, byte[] needle,
+ List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes) throws IOException {
+ int bufOffset = initBufOffset;
+ List<Map<String, Object>> matches = initialMatches;
+
+ byte[] newBeforeBytes;
+ Integer newByteOffset;
+
+ while (true) {
+ int offset = offsetOfBytes(haystack.array(), needle, bufOffset);
+ if (matches.size() < numMatches && offset >= 0) {
+ int fileOffset = offsetToBuf + offset;
+ int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length;
+
+ byte[] beforeArg = null;
+ byte[] afterArg = null;
+ if (offset < GREP_CONTEXT_SIZE) {
+ beforeArg = beforeBytes;
+ }
+
+ if (offset > bytesNeededAfterMatch) {
+ afterArg = tryReadAhead(stream, haystack, offset, fileLength, bytesRead);
+ }
+
+ bufOffset = offset + needle.length;
+ matches.add(mkMatchData(needle, haystack, offset, fileOffset,
+ file.getCanonicalPath(), isDaemon, beforeArg, afterArg));
+ } else {
+ int beforeStrToOffset = Math.min(haystack.limit(), GREP_MAX_SEARCH_SIZE);
+ int beforeStrFromOffset = Math.max(0, beforeStrToOffset - GREP_CONTEXT_SIZE);
+ newBeforeBytes = Arrays.copyOfRange(haystack.array(), beforeStrFromOffset, beforeStrToOffset);
+
+ // It's OK if new-byte-offset is negative.
+ // This is normal if we are out of bytes to read from a small file.
+ if (matches.size() >= numMatches) {
+ newByteOffset = ((Number) last(matches).get("byteOffset")).intValue() + needle.length;
+ } else {
+ newByteOffset = bytesSkipped + bytesRead - GREP_MAX_SEARCH_SIZE;
+ }
+
+ break;
+ }
+ }
+
+ return new SubstringSearchResult(matches, newByteOffset, newBeforeBytes);
+ }
+
+ private int rotateGrepBuffer(ByteBuffer buf, BufferedInputStream stream, int totalBytesRead, File file,
+ int fileLength) throws IOException {
+ byte[] bufArray = buf.array();
+
+ // Copy the 2nd half of the buffer to the first half.
+ System.arraycopy(bufArray, GREP_MAX_SEARCH_SIZE, bufArray, 0, GREP_MAX_SEARCH_SIZE);
+
+ // Zero-out the 2nd half to prevent accidental matches.
+ Arrays.fill(bufArray, GREP_MAX_SEARCH_SIZE, bufArray.length, (byte) 0);
+
+ // Fill the 2nd half with new bytes from the stream.
+ int bytesRead = stream.read(bufArray, GREP_MAX_SEARCH_SIZE, Math.min((int) fileLength, GREP_MAX_SEARCH_SIZE));
+ buf.limit(GREP_MAX_SEARCH_SIZE + bytesRead);
+ return totalBytesRead + bytesRead;
+ }
+
+
+ private Map<String, Object> mkMatchData(byte[] needle, ByteBuffer haystack, int haystackOffset, int fileOffset, String fname,
+ boolean isDaemon, byte[] beforeBytes, byte[] afterBytes)
+ throws UnsupportedEncodingException, UnknownHostException {
+ String url;
+ if (isDaemon) {
+ url = urlToMatchCenteredInLogPageDaemonFile(needle, fname, fileOffset, logviewerPort);
+ } else {
+ url = urlToMatchCenteredInLogPage(needle, fname, fileOffset, logviewerPort);
+ }
+
+ byte[] haystackBytes = haystack.array();
+ String beforeString;
+ String afterString;
+
+ if (haystackOffset >= GREP_CONTEXT_SIZE) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(new String(haystackBytes, (haystackOffset - GREP_CONTEXT_SIZE), GREP_CONTEXT_SIZE, "UTF-8"));
+ sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
+ beforeString = sb.toString();
+ } else {
+ int numDesired = Math.max(0, GREP_CONTEXT_SIZE - haystackOffset);
+ int beforeSize = beforeBytes != null ? beforeBytes.length : 0;
+ int numExpected = Math.min(beforeSize, numDesired);
+
+ if (numExpected > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(new String(beforeBytes, numExpected - beforeSize, numExpected, "UTF-8"));
+ sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
+ beforeString = sb.toString();
+ } else {
+ beforeString = new String(haystackBytes, 0, haystackOffset, "UTF-8");
+ }
+ }
+
+ int needleSize = needle.length;
+ int afterOffset = haystackOffset + needleSize;
+ int haystackSize = haystack.limit();
+
+ if ((afterOffset + GREP_CONTEXT_SIZE) < haystackSize) {
+ afterString = new String(haystackBytes, afterOffset, GREP_CONTEXT_SIZE, "UTF-8");
+ } else {
+ int numDesired = GREP_CONTEXT_SIZE - (haystackSize - afterOffset);
+ int afterSize = afterBytes != null ? afterBytes.length : 0;
+ int numExpected = Math.min(afterSize, numDesired);
+
+ if (numExpected > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8"));
+ sb.append(new String(afterBytes, 0, numExpected, "UTF-8"));
+ afterString = sb.toString();
+ } else {
+ afterString = new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8");
+ }
+ }
+
+ Map<String, Object> ret = new HashMap<>();
+ ret.put("byteOffset", fileOffset);
+ ret.put("beforeString", beforeString);
+ ret.put("afterString", afterString);
+ ret.put("matchString", new String(needle, "UTF-8"));
+ ret.put("logviewerURL", url);
+
+ return ret;
+ }
+
+ /**
+ * Tries once to read ahead in the stream to fill the context and
+ * resets the stream to its position before the call.
+ */
+ private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) throws IOException {
+ int numExpected = Math.min(fileLength - bytesRead, GREP_CONTEXT_SIZE);
+ byte[] afterBytes = new byte[numExpected];
+ stream.mark(numExpected);
+ // Only try reading once.
+ stream.read(afterBytes, 0, numExpected);
+ stream.reset();
+ return afterBytes;
+ }
+
+ /**
+ * Searches a given byte array for a match of a sub-array of bytes.
+ * Returns the offset to the byte that matches, or -1 if no match was found.
+ */
+ private int offsetOfBytes(byte[] buffer, byte[] search, int initOffset) {
+ if (search.length <= 0) {
+ throw new IllegalArgumentException("Search array should not be empty.");
+ }
+
+ if (initOffset < 0) {
+ throw new IllegalArgumentException("Start offset shouldn't be negative.");
+ }
+
+ int offset = initOffset;
+ int candidateOffset = initOffset;
+ int valOffset = 0;
+ int retOffset = 0;
+
+ while (true) {
+ if (search.length - valOffset <= 0) {
+ // found
+ retOffset = candidateOffset;
+ break;
+ } else {
+ if (offset >= buffer.length) {
+ // We ran out of buffer for the search.
+ retOffset = -1;
+ break;
+ } else {
+ if (search[valOffset] != buffer[offset]) {
+ // The match at this candidate offset failed, so start over with the
+ // next candidate byte from the buffer.
+ int newOffset = candidateOffset + 1;
+
+ offset = newOffset;
+ candidateOffset = newOffset;
+ valOffset = 0;
+ } else {
+ // So far it matches. Keep going...
+ offset = offset + 1;
+ valOffset = valOffset + 1;
+ }
+ }
+ }
+ }
+
+ return retOffset;
+ }
+
+ /**
+ * This response data only includes a next byte offset if there is more of the file to read.
+ */
+ private Map<String, Object> mkGrepResponse(byte[] searchBytes, Integer offset, List<Map<String, Object>> matches,
+ Integer nextByteOffset) throws UnsupportedEncodingException {
+ Map<String, Object> ret = new HashMap<>();
+ ret.put("searchString", new String(searchBytes, "UTF-8"));
+ ret.put("startByteOffset", offset);
+ ret.put("matches", matches);
+ if (nextByteOffset != null) {
+ ret.put("nextByteOffset", nextByteOffset);
+ }
+ return ret;
+ }
+
+ @VisibleForTesting
+ String urlToMatchCenteredInLogPage(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
+ String host = Utils.hostname();
+ String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
+ takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 3));
+
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("file", splittedFileName);
+ parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
+ parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE);
+
+ return URLBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
+ }
+
+ private String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
+ String host = Utils.hostname();
+ String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
+ takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 1));
+
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("file", splittedFileName);
+ parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
+ parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE);
+
+ return URLBuilder.build(String.format("http://%s:%d/api/v1/daemonlog", host, port), parameters);
+ }
+
+ @VisibleForTesting
+ public static class Matched implements JSONAware {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private int fileOffset;
+ private String searchString;
+ private List<Map<String, Object>> matches;
+
+ public Matched(int fileOffset, String searchString, List<Map<String, Object>> matches) {
+ this.fileOffset = fileOffset;
+ this.searchString = searchString;
+ this.matches = matches;
+ }
+
+ public int getFileOffset() {
+ return fileOffset;
+ }
+
+ public String getSearchString() {
+ return searchString;
+ }
+
+ public List<Map<String, Object>> getMatches() {
+ return matches;
+ }
+
+ @Override
+ public String toJSONString() {
+ try {
+ return OBJECT_MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class SubstringSearchResult {
+ private List<Map<String, Object>> matches;
+ private Integer newByteOffset;
+ private byte[] newBeforeBytes;
+
+ public SubstringSearchResult(List<Map<String, Object>> matches, Integer newByteOffset, byte[] newBeforeBytes) {
+ this.matches = matches;
+ this.newByteOffset = newByteOffset;
+ this.newBeforeBytes = newBeforeBytes;
+ }
+
+ public List<Map<String, Object>> getMatches() {
+ return matches;
+ }
+
+ public Integer getNewByteOffset() {
+ return newByteOffset;
+ }
+
+ public byte[] getNewBeforeBytes() {
+ return newBeforeBytes;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
new file mode 100644
index 0000000..39608da
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.handler;
+
+import j2html.tags.DomContent;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.utils.ServerUtils;
+
+import javax.ws.rs.core.Response;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static j2html.TagCreator.a;
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.head;
+import static j2html.TagCreator.html;
+import static j2html.TagCreator.li;
+import static j2html.TagCreator.link;
+import static j2html.TagCreator.title;
+import static j2html.TagCreator.ul;
+import static java.util.stream.Collectors.toList;
+
+public class LogviewerProfileHandler {
+
+ public static final String WORKER_LOG_FILENAME = "worker.log";
+ private final String logRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+
+ public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer) {
+ this.logRoot = logRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+ }
+
+ public Response listDumpFiles(String topologyId, String hostPort, String user) throws IOException {
+ String portStr = hostPort.split(":")[1];
+ File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
+
+ if (dir.exists()) {
+ String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
+ if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
+ String content = buildDumpFileListPage(topologyId, hostPort, dir);
+ return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+ private String buildDumpFileListPage(String topologyId, String hostPort, File dir) throws IOException {
+ List<DomContent> liTags = getProfilerDumpFiles(dir).stream()
+ .map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + file)))
+ .collect(toList());
+
+ return html(
+ head(
+ title("File Dumps - Storm Log Viewer"),
+ link().withRel("stylesheet").withHref("/css/bootstrap-3.3.1.min.css"),
+ link().withRel("stylesheet").withHref("/css/jquery.dataTables.1.10.4.min.css"),
+ link().withRel("stylesheet").withHref("/css/style.css")
+ ),
+ body(
+ ul(liTags.toArray(new DomContent[]{}))
+ )
+ ).render();
+ }
+
+ public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException {
+ String portStr = hostPort.split(":")[1];
+ File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
+ File file = new File(dir, fileName);
+
+ if (dir.exists() && file.exists()) {
+ String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
+ if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
+ return LogviewerResponseBuilder.buildDownloadFile(file);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+ private List<String> getProfilerDumpFiles(File dir) throws IOException {
+ List<File> filesForDir = DirectoryCleaner.getFilesForDir(dir);
+ return filesForDir.stream().filter(file -> {
+ String fileName = file.getName();
+ return StringUtils.isNotEmpty(fileName)
+ && (fileName.endsWith(".txt") || fileName.endsWith(".jfr") || fileName.endsWith(".bin"));
+ }).map(File::getName).collect(toList());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
new file mode 100644
index 0000000..c8a4f69
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.logviewer.utils;
+
+import java.io.IOException;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.DirectoryStream;
+import java.util.Stack;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.Comparator;
+import java.util.PriorityQueue;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Provide methods to help Logviewer to clean up
+ * files in directories and to get a list of files without
+ * worrying about excessive memory usage.
+ *
+ */
+public class DirectoryCleaner {
+ private static final Logger LOG = LoggerFactory.getLogger(DirectoryCleaner.class);
+ // used to recognize the pattern of active log files, we may remove the "current" from this list
+ private static final Pattern ACTIVE_LOG_PATTERN = Pattern.compile(".*\\.(log|err|out|current|yaml|pid)$");
+ // used to recognize the pattern of some meta files in a worker log directory
+ private static final Pattern META_LOG_PATTERN= Pattern.compile(".*\\.(yaml|pid)$");
+
+ // not defining this as static is to allow for mocking in tests
+ public DirectoryStream<Path> getStreamForDirectory(File dir) throws IOException {
+ DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath());
+ return stream;
+ }
+
+ /**
+ * If totalSize of files exceeds the either the per-worker quota or global quota,
+ * Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs.
+ * We use the parameter forPerDir to switch between the two deletion modes.
+ * @param dirs the list of directories to be scanned for deletion
+ * @param quota the per-dir quota or the total quota for the all directories
+ * @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories globally
+ * @param activeDirs only for global deletion, we want to skip the active logs in activeDirs
+ * @return number of files deleted
+ */
+ public int deleteOldestWhileTooLarge(List<File> dirs,
+ long quota, boolean forPerDir, Set<String> activeDirs) throws IOException {
+ final int PQ_SIZE = 1024; // max number of files to delete for every round
+ final int MAX_ROUNDS = 512; // max rounds of scanning the dirs
+ long totalSize = 0;
+ int deletedFiles = 0;
+
+ for (File dir : dirs) {
+ try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
+ for (Path path : stream) {
+ File file = path.toFile();
+ totalSize += file.length();
+ }
+ }
+ }
+
+ LOG.debug("totalSize: {} quota: {}", totalSize, quota);
+ long toDeleteSize = totalSize - quota;
+ if (toDeleteSize <= 0) {
+ return deletedFiles;
+ }
+
+ Comparator<File> comparator = new Comparator<File>() {
+ public int compare(File f1, File f2) {
+ if (f1.lastModified() > f2.lastModified()) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+ };
+ // the oldest pq_size files in this directory will be placed in PQ, with the newest at the root
+ PriorityQueue<File> pq = new PriorityQueue<File>(PQ_SIZE, comparator);
+ int round = 0;
+ while (toDeleteSize > 0) {
+ LOG.debug("To delete size is {}, start a new round of deletion, round: {}", toDeleteSize, round);
+ for (File dir : dirs) {
+ try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
+ for (Path path : stream) {
+ File file = path.toFile();
+ if (isFileEligibleToSkipDelete(forPerDir, activeDirs, dir, file)) {
+ continue;
+ }
+ if (pq.size() < PQ_SIZE) {
+ pq.offer(file);
+ } else {
+ if (file.lastModified() < pq.peek().lastModified()) {
+ pq.poll();
+ pq.offer(file);
+ }
+ }
+ }
+ }
+ }
+ // need to reverse the order of elements in PQ to delete files from oldest to newest
+ Stack<File> stack = new Stack<File>();
+ while (!pq.isEmpty()) {
+ File file = pq.poll();
+ stack.push(file);
+ }
+ while (!stack.isEmpty() && toDeleteSize > 0) {
+ File file = stack.pop();
+ toDeleteSize -= file.length();
+ LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getCanonicalPath(), file.length(), file.lastModified());
+ file.delete();
+ deletedFiles++;
+ }
+ pq.clear();
+ round++;
+ if (round >= MAX_ROUNDS) {
+ if (forPerDir) {
+ LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " +
+ "a single directory : {}, will delete the rest files in next interval.",
+ MAX_ROUNDS, dirs.get(0).getCanonicalPath());
+ } else {
+ LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " +
+ "will delete the rest files in next interval.", MAX_ROUNDS);
+ }
+ break;
+ }
+ }
+ return deletedFiles;
+ }
+
+ private boolean isFileEligibleToSkipDelete(boolean forPerDir, Set<String> activeDirs, File dir, File file) throws IOException {
+ if (forPerDir) {
+ if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
+ return true;
+ }
+ } else { // for global cleanup
+ if (activeDirs.contains(dir.getCanonicalPath())) { // for an active worker's dir, make sure for the last "/"
+ if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
+ return true;
+ }
+ } else {
+ if (META_LOG_PATTERN.matcher(file.getName()).matches()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ // Note that to avoid memory problem, we only return the first 1024 files in a directory
+ public static List<File> getFilesForDir(File dir) throws IOException {
+ List<File> files = new ArrayList<File>();
+ final int MAX_NUM = 1024;
+
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath())) {
+ for (Path path : stream) {
+ files.add(path.toFile());
+ if (files.size() >= MAX_NUM) {
+ break;
+ }
+ }
+ }
+ return files;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
new file mode 100644
index 0000000..cca2a52
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.IOUtils;
+import org.apache.storm.StormTimer;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+import org.jooq.lambda.tuple.Tuple2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.function.BinaryOperator;
+import java.util.stream.StreamSupport;
+
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB;
+
+public class LogCleaner implements Runnable, Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
+ public static final String WORKER_YAML = "worker.yaml";
+
+ private final Map<String, Object> stormConf;
+ private final Integer intervalSecs;
+ private final String logRootDir;
+ private final DirectoryCleaner directoryCleaner;
+
+ private StormTimer logviewerCleanupTimer;
+ private final long maxSumWorkerLogsSizeMb;
+ private long maxPerWorkerLogsSizeMb;
+
+ public LogCleaner(Map<String, Object> stormConf, DirectoryCleaner directoryCleaner) {
+ String logRootDir = ConfigUtils.workerArtifactsRoot(stormConf);
+
+ this.stormConf = stormConf;
+ this.intervalSecs = ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
+ this.logRootDir = logRootDir;
+ this.directoryCleaner = directoryCleaner;
+
+ maxSumWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB));
+ maxPerWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB));
+ maxPerWorkerLogsSizeMb = Math.min(maxPerWorkerLogsSizeMb, (long) (maxSumWorkerLogsSizeMb * 0.5));
+
+ LOG.info("configured max total size of worker logs: {} MB, max total size of worker logs per directory: {} MB",
+ maxSumWorkerLogsSizeMb, maxPerWorkerLogsSizeMb);
+ }
+
+ public void start() {
+ if (intervalSecs != null) {
+ LOG.debug("starting log cleanup thread at interval: {}", intervalSecs);
+
+ logviewerCleanupTimer = new StormTimer("logviewer-cleanup", (t, e) -> {
+ LOG.error("Error when doing logs cleanup", e);
+ Utils.exitProcess(20, "Error when doing log cleanup");
+ });
+
+ logviewerCleanupTimer.scheduleRecurring(0, intervalSecs, this);
+ } else {
+ LOG.warn("The interval for log cleanup is not set. Skip starting log cleanup thread.");
+ }
+
+ }
+
+ public void close() {
+ if (logviewerCleanupTimer != null) {
+ try {
+ logviewerCleanupTimer.close();
+ } catch (Exception ex) {
+ throw Utils.wrapInRuntime(ex);
+ }
+ }
+ }
+
+ /**
+ * Delete old log dirs for which the workers are no longer alive.
+ */
+ public void run() {
+ try {
+ int nowSecs = Time.currentTimeSecs();
+ Set<File> oldLogDirs = selectDirsForCleanup(nowSecs * 1000, logRootDir);
+
+ SortedSet<File> deadWorkerDirs = getDeadWorkerDirs(nowSecs, oldLogDirs);
+
+ LOG.debug("log cleanup: now={} old log dirs {} dead worker dirs {}", nowSecs,
+ oldLogDirs.stream().map(File::getName).collect(joining(",")),
+ deadWorkerDirs.stream().map(File::getName).collect(joining(",")));
+
+ deadWorkerDirs.forEach(Unchecked.consumer(dir -> {
+ String path = dir.getCanonicalPath();
+ LOG.info("Cleaning up: Removing {}", path);
+
+ try {
+ Utils.forceDelete(path);
+ cleanupEmptyTopoDirectory(dir);
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ }
+ }));
+
+ perWorkerDirCleanup(new File(logRootDir), maxPerWorkerLogsSizeMb * 1024 * 1024, directoryCleaner);
+ globalLogCleanup(new File(logRootDir), maxSumWorkerLogsSizeMb * 1024 * 1024, directoryCleaner);
+ } catch (Exception ex) {
+ LOG.error("Exception while cleaning up old log.", ex);
+ }
+ }
+
+ /**
+ * Delete the oldest files in each overloaded worker log dir.
+ */
+ @VisibleForTesting
+ List<Integer> perWorkerDirCleanup(File rootDir, long size, DirectoryCleaner cleaner) {
+ return WorkerLogs.getAllWorkerDirs(rootDir).stream()
+ .map(Unchecked.function(dir -> cleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null)))
+ .collect(toList());
+ }
+
+ /**
+ * Delete the oldest files in overloaded worker-artifacts globally.
+ */
+ @VisibleForTesting
+ int globalLogCleanup(File rootDir, long size, DirectoryCleaner cleaner) throws Exception {
+ List<File> workerDirs = new ArrayList<>(WorkerLogs.getAllWorkerDirs(rootDir));
+ Set<String> aliveWorkerDirs = new HashSet<>(getAliveWorkerDirs(rootDir));
+
+ return cleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs);
+ }
+
+ /**
+ * Return a sorted set of java.io.Files that were written by workers that are now active.
+ */
+ @VisibleForTesting
+ SortedSet<String> getAliveWorkerDirs(File rootDir) throws Exception {
+ Set<String> aliveIds = getAliveIds(Time.currentTimeSecs());
+ Set<File> logDirs = WorkerLogs.getAllWorkerDirs(rootDir);
+ Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
+
+ return idToDir.entrySet().stream()
+ .filter(entry -> aliveIds.contains(entry.getKey()))
+ .map(Unchecked.function(entry -> entry.getValue().getCanonicalPath()))
+ .collect(toCollection(TreeSet::new));
+ }
+
+ @VisibleForTesting
+ Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
+ return logDirs.stream().map(Unchecked.function(logDir -> {
+ Optional<File> metaFile = getMetadataFileForWorkerLogDir(logDir);
+
+ return metaFile.map(Unchecked.function(m -> new Tuple2<>(getWorkerIdFromMetadataFile(m.getCanonicalPath()), logDir)))
+ .orElse(new Tuple2<>("", logDir));
+ })).collect(toMap(Tuple2::v1, Tuple2::v2));
+ }
+
+ @VisibleForTesting
+ Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
+ File metaFile = new File(logDir, WORKER_YAML);
+ if (metaFile.exists()) {
+ return Optional.of(metaFile);
+ } else {
+ LOG.warn("Could not find {} to clean up for {}", metaFile.getCanonicalPath(), logDir);
+ return Optional.empty();
+ }
+ }
+
+ @VisibleForTesting
+ String getWorkerIdFromMetadataFile(String metaFile) {
+ Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
+ return ObjectReader.getString(map.get("worker-id"), null);
+ }
+
+ private Set<String> getAliveIds(int nowSecs) throws Exception {
+ return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
+ .filter(entry -> Objects.nonNull(entry.getValue())
+ && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf))
+ .map(Map.Entry::getKey)
+ .collect(toCollection(TreeSet::new));
+ }
+
+ /**
+ * Delete the topo dir if it contains zero port dirs.
+ */
+ @VisibleForTesting
+ void cleanupEmptyTopoDirectory(File dir) throws IOException {
+ File topoDir = dir.getParentFile();
+ if (topoDir.listFiles().length == 0) {
+ Utils.forceDelete(topoDir.getCanonicalPath());
+ }
+ }
+
+ /**
+ * Return a sorted set of java.io.Files that were written by workers that are now dead.
+ */
+ @VisibleForTesting
+ SortedSet<File> getDeadWorkerDirs(int nowSecs, Set<File> logDirs) throws Exception {
+ if (logDirs.isEmpty()) {
+ return new TreeSet<>();
+ } else {
+ Set<String> aliveIds = getAliveIds(nowSecs);
+ Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
+
+ return idToDir.entrySet().stream()
+ .filter(entry -> !aliveIds.contains(entry.getKey()))
+ .map(Map.Entry::getValue)
+ .collect(toCollection(TreeSet::new));
+ }
+ }
+
+ @VisibleForTesting
+ Set<File> selectDirsForCleanup(long nowMillis, String rootDir) {
+ FileFilter fileFilter = mkFileFilterForLogCleanup(nowMillis);
+
+ return Arrays.stream(new File(rootDir).listFiles())
+ .flatMap(topoDir -> Arrays.stream(topoDir.listFiles(fileFilter)))
+ .collect(toCollection(TreeSet::new));
+ }
+
+ @VisibleForTesting
+ FileFilter mkFileFilterForLogCleanup(long nowMillis) {
+ final long cutoffAgeMillis = cleanupCutoffAgeMillis(nowMillis);
+ return file -> !file.isFile() && lastModifiedTimeWorkerLogdir(file) <= cutoffAgeMillis;
+ }
+
+ /**
+ * Return the last modified time for all log files in a worker's log dir.
+ * Using stream rather than File.listFiles is to avoid large mem usage
+ * when a directory has too many files.
+ */
+ private long lastModifiedTimeWorkerLogdir(File logDir) {
+ long dirModified = logDir.lastModified();
+
+ DirectoryStream<Path> dirStream = null;
+ try {
+ dirStream = directoryCleaner.getStreamForDirectory(logDir);
+ } catch (IOException e) {
+ return dirModified;
+ }
+
+ if (dirStream == null) {
+ return dirModified;
+ }
+
+ try {
+ return StreamSupport.stream(dirStream.spliterator(), false)
+ .reduce(dirModified, (maximum, path) -> {
+ long curr = path.toFile().lastModified();
+ return curr > maximum ? curr : maximum;
+ }, BinaryOperator.maxBy(Long::compareTo));
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ return dirModified;
+ } finally {
+ if (DirectoryStream.class.isInstance(dirStream)) {
+ IOUtils.closeQuietly(dirStream);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ long cleanupCutoffAgeMillis(long nowMillis) {
+ return nowMillis - (ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_AGE_MINS)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
new file mode 100644
index 0000000..7d9cb50
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.utils;
+
+import javax.ws.rs.core.Response;
+import java.io.File;
+import java.io.IOException;
+
+public class LogFileDownloader {
+
+ private final String logRoot;
+ private final String daemonLogRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+
+ public LogFileDownloader(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
+ this.logRoot = logRoot;
+ this.daemonLogRoot = daemonLogRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+ }
+
+ public Response downloadFile(String fileName, String user, boolean isDaemon) throws IOException {
+ String rootDir = isDaemon ? daemonLogRoot : logRoot;
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ if (file.exists()) {
+ if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ return LogviewerResponseBuilder.buildDownloadFile(file);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
new file mode 100644
index 0000000..d0f2857
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.utils;
+
+import com.google.common.io.ByteStreams;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.ui.UIHelpers;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.h2;
+import static javax.ws.rs.core.Response.Status.OK;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+
+public class LogviewerResponseBuilder {
+
+ private LogviewerResponseBuilder() {
+ }
+
+ public static Response buildSuccessHtmlResponse(String content) {
+ return Response.status(OK).entity(content)
+ .type(MediaType.TEXT_HTML_TYPE).build();
+ }
+
+ public static Response buildSuccessJsonResponse(Object entity, String callback, String origin) {
+ return new JsonResponseBuilder().setData(entity).setCallback(callback)
+ .setHeaders(LogviewerResponseBuilder.getHeadersForSuccessResponse(origin)).build();
+ }
+
+ public static Response buildDownloadFile(File file) throws IOException {
+ // do not close this InputStream in method: it will be used from jetty server
+ InputStream is = new FileInputStream(file);
+ return Response.status(OK)
+ .entity(wrapWithStreamingOutput(is))
+ .type(MediaType.APPLICATION_OCTET_STREAM_TYPE)
+ .header("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"")
+ .build();
+ }
+
+ public static Response buildResponseUnautohrizedUser(String user) {
+ String entity = buildUnauthorizedUserHtml(user);
+ return Response.status(OK)
+ .entity(entity)
+ .type(MediaType.TEXT_HTML_TYPE)
+ .build();
+ }
+
+ public static Response buildResponsePageNotFound() {
+ return Response.status(404)
+ .entity("Page not found")
+ .type(MediaType.TEXT_HTML_TYPE)
+ .build();
+ }
+
+ public static Response buildUnauthorizedUserJsonResponse(String user, String callback) {
+ return new JsonResponseBuilder().setData(UIHelpers.unauthorizedUserJson(user))
+ .setCallback(callback).setStatus(401).build();
+ }
+
+ public static Response buildExceptionJsonResponse(Exception ex, String callback) {
+ return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(ex))
+ .setCallback(callback).setStatus(500).build();
+ }
+
+ private static Map<String, Object> getHeadersForSuccessResponse(String origin) {
+ Map<String, Object> headers = new HashMap<>();
+ headers.put("Access-Control-Allow-Origin", origin);
+ headers.put("Access-Control-Allow-Credentials", "true");
+ return headers;
+ }
+
+ private static String buildUnauthorizedUserHtml(String user) {
+ String content = "User '" + escapeHtml(user) + "' is not authorized.";
+ return body(h2(content)).render();
+ }
+
+ private static StreamingOutput wrapWithStreamingOutput(final InputStream inputStream) {
+ return os -> {
+ OutputStream wrappedOutputStream = os;
+ if (!(os instanceof BufferedOutputStream)) {
+ wrappedOutputStream = new BufferedOutputStream(os);
+ }
+
+ ByteStreams.copy(inputStream, wrappedOutputStream);
+
+ wrappedOutputStream.flush();
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
new file mode 100644
index 0000000..55691b2
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.security.auth.AuthUtils;
+import org.apache.storm.security.auth.IGroupMappingServiceProvider;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerConfigUtils;
+import org.apache.storm.utils.Utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.storm.DaemonConfig.UI_FILTER;
+
+public class ResourceAuthorizer {
+
+ private final Map<String, Object> stormConf;
+ private final IGroupMappingServiceProvider groupMappingServiceProvider;
+
+ public ResourceAuthorizer(Map<String, Object> stormConf) {
+ this.stormConf = stormConf;
+ this.groupMappingServiceProvider = AuthUtils.GetGroupMappingServiceProviderPlugin(stormConf);
+ }
+
+ public boolean isUserAllowedToAccessFile(String fileName, String user) {
+ return isUiFilterNotSet() || isAuthorizedLogUser(user, fileName);
+ }
+
+ public boolean isAuthorizedLogUser(String user, String fileName) {
+ if (StringUtils.isEmpty(user) || StringUtils.isEmpty(fileName)
+ || getLogUserGroupWhitelist(fileName) == null) {
+ return false;
+ } else {
+ Set<String> groups = getUserGroups(user);
+ LogUserGroupWhitelist whitelist = getLogUserGroupWhitelist(fileName);
+
+ List<String> logsUsers = new ArrayList<>();
+ logsUsers.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_USERS)));
+ logsUsers.addAll(ObjectReader.getStrings(stormConf.get(Config.NIMBUS_ADMINS)));
+ logsUsers.addAll(whitelist.getUserWhitelist());
+
+ List<String> logsGroups = new ArrayList<>();
+ logsGroups.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_GROUPS)));
+ logsGroups.addAll(whitelist.getGroupWhitelist());
+
+ return logsUsers.stream().anyMatch(u -> u.equals(user)) ||
+ Sets.intersection(groups, new HashSet<>(logsGroups)).size() > 0;
+ }
+ }
+
+ public LogUserGroupWhitelist getLogUserGroupWhitelist(String fileName) {
+ File wlFile = ServerConfigUtils.getLogMetaDataFile(fileName);
+ Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(wlFile.getAbsolutePath());
+
+ if (map == null) {
+ return null;
+ }
+
+ List<String> logsUsers = ObjectReader.getStrings(map.get(DaemonConfig.LOGS_USERS));
+ List<String> logsGroups = ObjectReader.getStrings(map.get(DaemonConfig.LOGS_GROUPS));
+ return new LogUserGroupWhitelist(
+ logsUsers.isEmpty() ? new HashSet<>() : new HashSet<>(logsUsers),
+ logsGroups.isEmpty() ? new HashSet<>() : new HashSet<>(logsGroups)
+ );
+ }
+
+ @VisibleForTesting
+ Set<String> getUserGroups(String user) {
+ try {
+ if (StringUtils.isEmpty(user)) {
+ return new HashSet<>();
+ } else {
+ return groupMappingServiceProvider.getGroups(user);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private boolean isUiFilterNotSet() {
+ return StringUtils.isBlank(ObjectReader.getString(stormConf.get(UI_FILTER), null));
+ }
+
+ public static class LogUserGroupWhitelist {
+
+ private Set<String> userWhitelist;
+ private Set<String> groupWhitelist;
+
+ public LogUserGroupWhitelist(Set<String> userWhitelist, Set<String> groupWhitelist) {
+ this.userWhitelist = userWhitelist;
+ this.groupWhitelist = groupWhitelist;
+ }
+
+ public Set<String> getUserWhitelist() {
+ return userWhitelist;
+ }
+
+ public Set<String> getGroupWhitelist() {
+ return groupWhitelist;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
new file mode 100644
index 0000000..1d70b31
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
@@ -0,0 +1,60 @@
+package org.apache.storm.daemon.logviewer.utils;
+
+import org.apache.storm.utils.Utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toCollection;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+
+public class WorkerLogs {
+
+ private WorkerLogs() {
+ }
+
+ /**
+ * Return the path of the worker log with the format of topoId/port/worker.log.*
+ */
+ public static String getTopologyPortWorkerLog(File file) {
+ try {
+ String[] splitted = file.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR);
+ List<String> split = takeLast(Arrays.asList(splitted), 3);
+
+ return String.join(Utils.FILE_PATH_SEPARATOR, split);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static List<File> getAllLogsForRootDir(File logDir) throws IOException {
+ List<File> files = new ArrayList<>();
+ Set<File> topoDirFiles = getAllWorkerDirs(logDir);
+ if (topoDirFiles != null) {
+ for (File portDir : topoDirFiles) {
+ files.addAll(DirectoryCleaner.getFilesForDir(portDir));
+ }
+ }
+
+ return files;
+ }
+
+ public static Set<File> getAllWorkerDirs(File rootDir) {
+ File[] rootDirFiles = rootDir.listFiles();
+ if (rootDirFiles != null) {
+ return Arrays.stream(rootDirFiles).flatMap(topoDir -> {
+ File[] topoFiles = topoDir.listFiles();
+ return topoFiles != null ? Arrays.stream(topoFiles) : Stream.empty();
+ }).collect(toCollection(TreeSet::new));
+ }
+
+ return new TreeSet<>();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
new file mode 100644
index 0000000..cbf478c
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.webapp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
+import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
+import org.apache.storm.security.auth.AuthUtils;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ObjectReader;
+
+import javax.ws.rs.ApplicationPath;
+import javax.ws.rs.core.Application;
+import java.io.File;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME;
+
+@ApplicationPath("")
+public class LogviewerApplication extends Application {
+ private static Map<String, Object> stormConf;
+ private final Set<Object> singletons = new HashSet<Object>();
+
+ /**
+ * Constructor.
+ */
+ public LogviewerApplication() {
+ String logRoot = ConfigUtils.workerArtifactsRoot(stormConf);
+ String daemonLogRoot = logRootDir(ObjectReader.getString(stormConf.get(LOGVIEWER_APPENDER_NAME)));
+
+ ResourceAuthorizer resourceAuthorizer = new ResourceAuthorizer(stormConf);
+
+ LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, resourceAuthorizer);
+ LogviewerProfileHandler profileHandler = new LogviewerProfileHandler(logRoot, resourceAuthorizer);
+ LogviewerLogDownloadHandler logDownloadHandler = new LogviewerLogDownloadHandler(logRoot, daemonLogRoot,
+ resourceAuthorizer);
+ LogviewerLogSearchHandler logSearchHandler = new LogviewerLogSearchHandler(stormConf, logRoot, daemonLogRoot,
+ resourceAuthorizer);
+ IHttpCredentialsPlugin httpCredsHandler = AuthUtils.GetUiHttpCredentialsPlugin(stormConf);
+
+ singletons.add(new LogviewerResource(logviewer, profileHandler, logDownloadHandler, logSearchHandler, httpCredsHandler));
+ singletons.add(new AuthorizationExceptionMapper());
+ }
+
+ @Override
+ public Set<Object> getSingletons() {
+ return singletons;
+ }
+
+ public static void setup(Map<String, Object> stormConf) {
+ LogviewerApplication.stormConf = stormConf;
+ }
+
+ /**
+ * Given an appender name, as configured, get the parent directory of the appender's log file.
+ * Note that if anything goes wrong, this will throw an Error and exit.
+ */
+ private String logRootDir(String appenderName) {
+ Appender appender = ((LoggerContext) LogManager.getContext()).getConfiguration().getAppender(appenderName);
+ if (appenderName != null && appender != null && RollingFileAppender.class.isInstance(appender)) {
+ return new File(((RollingFileAppender) appender).getFileName()).getParent();
+ } else {
+ throw new RuntimeException("Log viewer could not find configured appender, or the appender is not a FileAppender. "
+ + "Please check that the appender name configured in storm and log4j agree.");
+ }
+ }
+
+}
[13/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* address checkstyle violations
* also addressed this from DRPC so far, but needs to decide for DRPC vs Drpc
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6a041a98
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6a041a98
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6a041a98
Branch: refs/heads/master
Commit: 6a041a98fc7fe04db3f520fa283192d96ffb0a56
Parents: 7f84691
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Jul 17 13:35:51 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Jul 17 13:35:51 2017 +0900
----------------------------------------------------------------------
pom.xml | 6 +
storm-buildtools/storm_checkstyle.xml | 6 +
storm-webapp/pom.xml | 2 +-
.../daemon/common/JsonResponseBuilder.java | 14 +-
.../daemon/drpc/webapp/DRPCApplication.java | 2 +-
.../daemon/logviewer/LogviewerConstant.java | 8 +-
.../storm/daemon/logviewer/LogviewerServer.java | 20 ++-
.../handler/LogviewerLogDownloadHandler.java | 30 +++-
.../handler/LogviewerLogPageHandler.java | 116 ++++++++++----
.../handler/LogviewerLogSearchHandler.java | 144 +++++++++++------
.../handler/LogviewerProfileHandler.java | 83 ++++++----
.../logviewer/utils/DirectoryCleaner.java | 51 ++++---
.../daemon/logviewer/utils/LogCleaner.java | 64 ++++----
.../logviewer/utils/LogFileDownloader.java | 18 ++-
.../utils/LogviewerResponseBuilder.java | 59 +++++--
.../logviewer/utils/ResourceAuthorizer.java | 48 ++++--
.../daemon/logviewer/utils/WorkerLogs.java | 82 +++++++---
.../logviewer/webapp/LogviewerApplication.java | 28 ++--
.../logviewer/webapp/LogviewerResource.java | 77 ++++++++--
.../daemon/utils/ListFunctionalSupport.java | 53 ++++++-
.../apache/storm/daemon/utils/StreamUtil.java | 9 ++
.../apache/storm/daemon/utils/URLBuilder.java | 43 ------
.../apache/storm/daemon/utils/UrlBuilder.java | 53 +++++++
.../storm/daemon/drpc/DRPCServerTest.java | 13 +-
.../storm/daemon/logviewer/LogviewerTest.java | 51 -------
.../handler/LogviewerLogPageHandlerTest.java | 32 ++--
.../handler/LogviewerLogSearchHandlerTest.java | 153 ++++++++++---------
.../testsupport/ArgumentsVerifier.java | 15 +-
.../testsupport/MockDirectoryBuilder.java | 12 +-
.../logviewer/testsupport/MockFileBuilder.java | 12 +-
.../daemon/logviewer/utils/LogCleanerTest.java | 61 ++++----
.../logviewer/utils/ResourceAuthorizerTest.java | 24 +--
.../daemon/logviewer/utils/WorkerLogsTest.java | 18 +--
33 files changed, 912 insertions(+), 495 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ffee0cd..9d890d9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -486,6 +486,12 @@
<exclude>**/derby.log</exclude>
<exclude>**/src/dev/**</exclude>
+ <!-- webapp test resources -->
+ <exclude>**/src/test/resources/test-3072.log.test</exclude>
+ <exclude>**/src/test/resources/small-worker.log.test</exclude>
+ <exclude>**/src/test/resources/test-worker.log.test</exclude>
+ <exclude>**/src/test/resources/logviewer-search-context-tests.log.test</exclude>
+
<!-- StormSQL -->
<exclude>**/src/codegen/config.fmpp</exclude>
<exclude>**/src/codegen/data/Parser.tdd</exclude>
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-buildtools/storm_checkstyle.xml
----------------------------------------------------------------------
diff --git a/storm-buildtools/storm_checkstyle.xml b/storm-buildtools/storm_checkstyle.xml
index 248e3ce..3b91be9 100644
--- a/storm-buildtools/storm_checkstyle.xml
+++ b/storm-buildtools/storm_checkstyle.xml
@@ -46,6 +46,9 @@
-->
<module name = "Checker">
+ <!-- Filter out Checkstyle warnings that have been suppressed with the @SuppressWarnings annotation -->
+ <module name="SuppressWarningsFilter" />
+
<property name="charset" value="UTF-8"/>
<property name="severity" value="warning"/>
@@ -58,6 +61,9 @@
</module>
<module name="TreeWalker">
+ <!-- Make the @SuppressWarnings annotations available to Checkstyle -->
+ <module name="SuppressWarningsHolder" />
+
<module name="OuterTypeFilename"/>
<module name="IllegalTokenText">
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/storm-webapp/pom.xml b/storm-webapp/pom.xml
index eeb6ee6..57ccb88 100644
--- a/storm-webapp/pom.xml
+++ b/storm-webapp/pom.xml
@@ -143,7 +143,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>500</maxAllowedViolations>
+ <maxAllowedViolations>4</maxAllowedViolations>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
index 2048d81..dc39bc6 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
@@ -18,12 +18,15 @@
package org.apache.storm.daemon.common;
-import org.apache.storm.ui.UIHelpers;
-
-import javax.ws.rs.core.Response;
import java.util.Collections;
import java.util.Map;
+import javax.ws.rs.core.Response;
+import org.apache.storm.ui.UIHelpers;
+
+/**
+ * Response builder for JSON. It utilizes {@link UIHelpers} to construct JSON body and headers.
+ */
public class JsonResponseBuilder {
private Object data;
private String callback;
@@ -56,6 +59,11 @@ public class JsonResponseBuilder {
return this;
}
+ /**
+ * Build a Response based on given parameters.
+ *
+ * @return A Response.
+ */
public Response build() {
String body = UIHelpers.getJsonResponseBody(data, callback, needSerialize);
Map<String, Object> respHeaders = UIHelpers.getJsonResponseHeaders(callback, headers);
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
index db42310..15cc4d7 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
@@ -24,8 +24,8 @@ import java.util.Set;
import javax.ws.rs.ApplicationPath;
import javax.ws.rs.core.Application;
-import org.apache.storm.daemon.drpc.DRPC;
import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
+import org.apache.storm.daemon.drpc.DRPC;
@ApplicationPath("")
public class DRPCApplication extends Application {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
index 612e0e2..d1a15be 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerConstant.java
@@ -18,6 +18,12 @@
package org.apache.storm.daemon.logviewer;
-public class LogviewerConstant {
+/**
+ * Constants which are used across logviewer related classes.
+ */
+public final class LogviewerConstant {
+ private LogviewerConstant() {
+ }
+
public static final int DEFAULT_BYTES_PER_PAGE = 51200;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
index 1b53809..4f9834d 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
@@ -18,8 +18,17 @@
package org.apache.storm.daemon.logviewer;
+import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES;
+
import com.codahale.metrics.Meter;
import com.google.common.annotations.VisibleForTesting;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
import org.apache.storm.daemon.logviewer.utils.LogCleaner;
@@ -40,14 +49,9 @@ import org.glassfish.jersey.servlet.ServletContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES;
-
+/**
+ * The main entry of Logviewer.
+ */
public class LogviewerServer implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(LogviewerServer.class);
private static final Meter meterShutdownCalls = StormMetricsRegistry.registerMeter("drpc:num-shutdown-calls");
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
index 7529297..7b2fbcb 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java
@@ -18,28 +18,52 @@
package org.apache.storm.daemon.logviewer.handler;
+import java.io.IOException;
+import javax.ws.rs.core.Response;
+
import org.apache.storm.daemon.logviewer.utils.LogFileDownloader;
import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-
public class LogviewerLogDownloadHandler {
private WorkerLogs workerLogs;
private final LogFileDownloader logFileDownloadHelper;
+ /**
+ * Constructor.
+ *
+ * @param logRoot root worker log directory
+ * @param daemonLogRoot root daemon log directory
+ * @param workerLogs {@link WorkerLogs}
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, WorkerLogs workerLogs, ResourceAuthorizer resourceAuthorizer) {
this.workerLogs = workerLogs;
this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, resourceAuthorizer);
}
+ /**
+ * Download an worker log.
+ *
+ * @param fileName file to download
+ * @param user username
+ * @return a Response which lets browsers download that file.
+ * @see {@link LogFileDownloader#downloadFile(String, String, boolean)}
+ */
public Response downloadLogFile(String fileName, String user) throws IOException {
workerLogs.setLogFilePermission(fileName);
return logFileDownloadHelper.downloadFile(fileName, user, false);
}
+ /**
+ * Download a daemon log.
+ *
+ * @param fileName file to download
+ * @param user username
+ * @return a Response which lets browsers download that file.
+ * @see {@link LogFileDownloader#downloadFile(String, String, boolean)}
+ */
public Response downloadDaemonLogFile(String fileName, String user) throws IOException {
return logFileDownloadHelper.downloadFile(fileName, user, true);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
index d2488f6..04b1a13 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
@@ -18,22 +18,26 @@
package org.apache.storm.daemon.logviewer.handler;
+import static j2html.TagCreator.a;
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.div;
+import static j2html.TagCreator.form;
+import static j2html.TagCreator.h3;
+import static j2html.TagCreator.head;
+import static j2html.TagCreator.html;
+import static j2html.TagCreator.input;
+import static j2html.TagCreator.link;
+import static j2html.TagCreator.p;
+import static j2html.TagCreator.pre;
+import static j2html.TagCreator.select;
+import static j2html.TagCreator.text;
+import static j2html.TagCreator.title;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+
import j2html.TagCreator;
import j2html.tags.DomContent;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
-import org.apache.storm.daemon.logviewer.LogviewerConstant;
-import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
-import org.apache.storm.daemon.utils.StreamUtil;
-import org.apache.storm.daemon.utils.URLBuilder;
-import org.apache.storm.ui.InvalidRequestException;
-import org.apache.storm.ui.UIHelpers;
-import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ServerUtils;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -54,10 +58,20 @@ import java.util.zip.GZIPInputStream;
import javax.ws.rs.core.Response;
-import static j2html.TagCreator.*;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toList;
-import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.logviewer.LogviewerConstant;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.UrlBuilder;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
public class LogviewerLogPageHandler {
private final String logRoot;
@@ -65,6 +79,14 @@ public class LogviewerLogPageHandler {
private final WorkerLogs workerLogs;
private final ResourceAuthorizer resourceAuthorizer;
+ /**
+ * Constructor.
+ *
+ * @param logRoot root worker log directory
+ * @param daemonLogRoot root daemon log directory
+ * @param workerLogs {@link WorkerLogs}
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerLogPageHandler(String logRoot, String daemonLogRoot,
WorkerLogs workerLogs,
ResourceAuthorizer resourceAuthorizer) {
@@ -74,6 +96,16 @@ public class LogviewerLogPageHandler {
this.resourceAuthorizer = resourceAuthorizer;
}
+ /**
+ * Enumerate worker log files for given criteria.
+ *
+ * @param user username
+ * @param port worker's port, null for all workers
+ * @param topologyId topology ID, null for all topologies
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @return list of worker logs for given criteria
+ */
public Response listLogFiles(String user, Integer port, String topologyId, String callback, String origin) throws IOException {
List<File> fileResults = null;
if (topologyId == null) {
@@ -130,7 +162,18 @@ public class LogviewerLogPageHandler {
return LogviewerResponseBuilder.buildSuccessJsonResponse(files, callback, origin);
}
- public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ /**
+ * Provides a worker log file to view.
+ *
+ * @param fileName file to view
+ * @param start start offset, can be null
+ * @param length length to read in this page, can be null
+ * @param grep search string if request is a result of the search, can be null
+ * @param user username
+ * @return HTML view page of worker log
+ */
+ public Response logPage(String fileName, Integer start, Integer length, String grep, String user)
+ throws IOException, InvalidRequestException {
String rootDir = logRoot;
if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
workerLogs.setLogFilePermission(fileName);
@@ -141,8 +184,6 @@ public class LogviewerLogPageHandler {
File topoDir = file.getParentFile().getParentFile();
if (file.exists() && new File(rootDir).getCanonicalFile().equals(topoDir.getParentFile())) {
- long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
-
SortedSet<File> logFiles;
try {
logFiles = Arrays.stream(topoDir.listFiles())
@@ -169,6 +210,7 @@ public class LogviewerLogPageHandler {
logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
}
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
start = start != null ? start : Long.valueOf(fileLength - length).intValue();
List<DomContent> bodyContents = new ArrayList<>();
@@ -209,15 +251,24 @@ public class LogviewerLogPageHandler {
}
}
- public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ /**
+ * Provides a daemon log file to view.
+ *
+ * @param fileName file to view
+ * @param start start offset, can be null
+ * @param length length to read in this page, can be null
+ * @param grep search string if request is a result of the search, can be null
+ * @param user username
+ * @return HTML view page of daemon log
+ */
+ public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user)
+ throws IOException, InvalidRequestException {
String rootDir = daemonLogRoot;
File file = new File(rootDir, fileName).getCanonicalFile();
String path = file.getCanonicalPath();
boolean isZipFile = path.endsWith(".gz");
if (file.exists() && new File(rootDir).getCanonicalFile().equals(file.getParentFile())) {
- long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
-
// all types of files included
List<File> logFiles = Arrays.stream(new File(rootDir).listFiles())
.filter(File::isFile)
@@ -239,6 +290,7 @@ public class LogviewerLogPageHandler {
logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
}
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
start = start != null ? start : Long.valueOf(fileLength - length).intValue();
List<DomContent> bodyContents = new ArrayList<>();
@@ -331,36 +383,36 @@ public class LogviewerLogPageHandler {
}
private DomContent pagerLinks(String fileName, Integer start, Integer length, Integer fileLength, String type) {
- int prevStart = Math.max(0, start - length);
- int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length;
- List<DomContent> btnLinks = new ArrayList<>();
-
Map<String, Object> urlQueryParams = new HashMap<>();
urlQueryParams.put("file", fileName);
urlQueryParams.put("start", Math.max(0, start - length));
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start));
+ List<DomContent> btnLinks = new ArrayList<>();
+
+ int prevStart = Math.max(0, start - length);
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start));
urlQueryParams.clear();
urlQueryParams.put("file", fileName);
urlQueryParams.put("start", 0);
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "First"));
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "First"));
urlQueryParams.clear();
urlQueryParams.put("file", fileName);
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Last"));
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Last"));
urlQueryParams.clear();
urlQueryParams.put("file", fileName);
urlQueryParams.put("start", Math.min(Math.max(0, fileLength - length), start + length));
urlQueryParams.put("length", length);
- btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start));
+ int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length;
+ btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start));
return div(btnLinks.toArray(new DomContent[]{}));
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
index 304c2d6..76f1683 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
@@ -18,29 +18,17 @@
package org.apache.storm.daemon.logviewer.handler;
+import static java.util.stream.Collectors.toList;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.first;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.last;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.BooleanUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
-import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.daemon.logviewer.LogviewerConstant;
-import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
-import org.apache.storm.daemon.utils.StreamUtil;
-import org.apache.storm.daemon.utils.URLBuilder;
-import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.ui.InvalidRequestException;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ServerUtils;
-import org.apache.storm.utils.Utils;
-import org.json.simple.JSONAware;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import javax.ws.rs.core.Response;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -58,15 +46,29 @@ import java.util.Map;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
-import static java.util.stream.Collectors.toList;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.first;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.last;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.daemon.logviewer.LogviewerConstant;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.UrlBuilder;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class LogviewerLogSearchHandler {
- private final static Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class);
+ private static final Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class);
public static final int GREP_MAX_SEARCH_SIZE = 1024;
public static final int GREP_BUF_SIZE = 2048;
@@ -79,6 +81,14 @@ public class LogviewerLogSearchHandler {
private final ResourceAuthorizer resourceAuthorizer;
private final Integer logviewerPort;
+ /**
+ * Constructor.
+ *
+ * @param stormConf storm configuration
+ * @param logRoot log root directory
+ * @param daemonLogRoot daemon log root directory
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerLogSearchHandler(Map<String, Object> stormConf, String logRoot, String daemonLogRoot,
ResourceAuthorizer resourceAuthorizer) {
this.stormConf = stormConf;
@@ -89,6 +99,19 @@ public class LogviewerLogSearchHandler {
this.logviewerPort = ObjectReader.getInt(stormConf.get(DaemonConfig.LOGVIEWER_PORT));
}
+ /**
+ * Search from a worker log file.
+ *
+ * @param fileName log file
+ * @param user username
+ * @param isDaemon whether the log file is regarding worker or daemon
+ * @param search search string
+ * @param numMatchesStr the count of maximum matches
+ * @param offsetStr start offset for log file
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @return Response containing JSON content representing search result
+ */
public Response searchLogFile(String fileName, String user, boolean isDaemon, String search,
String numMatchesStr, String offsetStr, String callback, String origin)
throws IOException, InvalidRequestException {
@@ -130,6 +153,21 @@ public class LogviewerLogSearchHandler {
return response;
}
+ /**
+ * Deep search across worker log files in a topology.
+ *
+ * @param topologyId topology ID
+ * @param user username
+ * @param search search string
+ * @param numMatchesStr the count of maximum matches
+ * @param portStr worker port, null or '*' if the request wants to search from all worker logs
+ * @param fileOffsetStr index (offset) of the log files
+ * @param offsetStr start offset for log file
+ * @param searchArchived true if the request wants to search also archived files, false if not
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @return Response containing JSON content representing search result
+ */
public Response deepSearchLogsForTopology(String topologyId, String user, String search,
String numMatchesStr, String portStr, String fileOffsetStr, String offsetStr,
Boolean searchArchived, String callback, String origin) {
@@ -178,8 +216,8 @@ public class LogviewerLogSearchHandler {
if (!containsPort) {
returnValue = new ArrayList<>();
} else {
- File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId +
- Utils.FILE_PATH_SEPARATOR + port);
+ File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId
+ + Utils.FILE_PATH_SEPARATOR + port);
if (!portDir.exists() || logsForPort(user, portDir).isEmpty()) {
returnValue = new ArrayList<>();
@@ -218,22 +256,19 @@ public class LogviewerLogSearchHandler {
}
@VisibleForTesting
- Map<String,Object> substringSearchDaemonLog(File file, String searchString) throws InvalidRequestException {
- return substringSearch(file, searchString, true, 10, 0);
- }
-
- @VisibleForTesting
Map<String,Object> substringSearch(File file, String searchString, int numMatches, int startByteOffset) throws InvalidRequestException {
return substringSearch(file, searchString, false, numMatches, startByteOffset);
}
- private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches, Integer startByteOffset) throws InvalidRequestException {
+ private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches,
+ Integer startByteOffset) throws InvalidRequestException {
try {
if (StringUtils.isEmpty(searchString)) {
throw new IllegalArgumentException("Precondition fails: search string should not be empty.");
}
if (searchString.getBytes("UTF-8").length > GREP_MAX_SEARCH_SIZE) {
- throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + GREP_MAX_SEARCH_SIZE);
+ throw new IllegalArgumentException("Precondition fails: the length of search string should be less than "
+ + GREP_MAX_SEARCH_SIZE);
}
boolean isZipFile = file.getName().endsWith(".gz");
@@ -255,9 +290,8 @@ public class LogviewerLogSearchHandler {
}
ByteBuffer buf = ByteBuffer.allocate(GREP_BUF_SIZE);
- byte[] bufArray = buf.array();
- int totalBytesRead = 0;
- byte[] searchBytes = searchString.getBytes("UTF-8");
+ final byte[] bufArray = buf.array();
+ final byte[] searchBytes = searchString.getBytes("UTF-8");
numMatches = numMatches != null ? numMatches : 10;
startByteOffset = startByteOffset != null ? startByteOffset : 0;
@@ -273,6 +307,7 @@ public class LogviewerLogSearchHandler {
Arrays.fill(bufArray, (byte) 0);
+ int totalBytesRead = 0;
int bytesRead = stream.read(bufArray, 0, Math.min((int) fileLength, GREP_BUF_SIZE));
buf.limit(bytesRead);
totalBytesRead += bytesRead;
@@ -295,7 +330,7 @@ public class LogviewerLogSearchHandler {
// The start index is positioned to find any possible
// occurrence search string that did not quite fit in the
// buffer on the previous read.
- int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length;
+ final int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length;
totalBytesRead = rotateGrepBuffer(buf, stream, totalBytesRead, file, fileLength);
if (totalBytesRead < 0) {
@@ -326,6 +361,10 @@ public class LogviewerLogSearchHandler {
}
}
+ @VisibleForTesting
+ Map<String,Object> substringSearchDaemonLog(File file, String searchString) throws InvalidRequestException {
+ return substringSearch(file, searchString, true, 10, 0);
+ }
/**
* Get the filtered, authorized, sorted log files for a port.
@@ -370,7 +409,7 @@ public class LogviewerLogSearchHandler {
String fileName = WorkerLogs.getTopologyPortWorkerLog(firstLog);
- List<Map<String, Object>> newMatches = new ArrayList<>(matches);
+ final List<Map<String, Object>> newMatches = new ArrayList<>(matches);
Map<String, Object> currentFileMatch = new HashMap<>(theseMatches);
currentFileMatch.put("fileName", fileName);
List<String> splitPath;
@@ -412,7 +451,8 @@ public class LogviewerLogSearchHandler {
private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, File file, int fileLength, int offsetToBuf,
int initBufOffset, BufferedInputStream stream, Integer bytesSkipped,
int bytesRead, ByteBuffer haystack, byte[] needle,
- List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes) throws IOException {
+ List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes)
+ throws IOException {
int bufOffset = initBufOffset;
List<Map<String, Object>> matches = initialMatches;
@@ -422,8 +462,8 @@ public class LogviewerLogSearchHandler {
while (true) {
int offset = offsetOfBytes(haystack.array(), needle, bufOffset);
if (matches.size() < numMatches && offset >= 0) {
- int fileOffset = offsetToBuf + offset;
- int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length;
+ final int fileOffset = offsetToBuf + offset;
+ final int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length;
byte[] beforeArg = null;
byte[] afterArg = null;
@@ -541,7 +581,8 @@ public class LogviewerLogSearchHandler {
* Tries once to read ahead in the stream to fill the context and
* resets the stream to its position before the call.
*/
- private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) throws IOException {
+ private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead)
+ throws IOException {
int numExpected = Math.min(fileLength - bytesRead, GREP_CONTEXT_SIZE);
byte[] afterBytes = new byte[numExpected];
stream.mark(numExpected);
@@ -617,7 +658,7 @@ public class LogviewerLogSearchHandler {
@VisibleForTesting
String urlToMatchCenteredInLogPage(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
- String host = Utils.hostname();
+ final String host = Utils.hostname();
String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 3));
@@ -626,12 +667,12 @@ public class LogviewerLogSearchHandler {
parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE);
- return URLBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
+ return UrlBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
}
@VisibleForTesting
String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
- String host = Utils.hostname();
+ final String host = Utils.hostname();
String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 1));
@@ -640,7 +681,7 @@ public class LogviewerLogSearchHandler {
parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE);
- return URLBuilder.build(String.format("http://%s:%d/api/v1/daemonlog", host, port), parameters);
+ return UrlBuilder.build(String.format("http://%s:%d/api/v1/daemonlog", host, port), parameters);
}
@VisibleForTesting
@@ -651,6 +692,13 @@ public class LogviewerLogSearchHandler {
private String searchString;
private List<Map<String, Object>> matches;
+ /**
+ * Constructor.
+ *
+ * @param fileOffset offset (index) of the files
+ * @param searchString search string
+ * @param matches map representing matched search result
+ */
public Matched(int fileOffset, String searchString, List<Map<String, Object>> matches) {
this.fileOffset = fileOffset;
this.searchString = searchString;
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
index 39608da..d7487e7 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java
@@ -18,18 +18,6 @@
package org.apache.storm.daemon.logviewer.handler;
-import j2html.tags.DomContent;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
-import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.utils.ServerUtils;
-
-import javax.ws.rs.core.Response;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
import static j2html.TagCreator.a;
import static j2html.TagCreator.body;
import static j2html.TagCreator.head;
@@ -40,17 +28,44 @@ import static j2html.TagCreator.title;
import static j2html.TagCreator.ul;
import static java.util.stream.Collectors.toList;
+import j2html.tags.DomContent;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.utils.ServerUtils;
+
public class LogviewerProfileHandler {
public static final String WORKER_LOG_FILENAME = "worker.log";
private final String logRoot;
private final ResourceAuthorizer resourceAuthorizer;
+ /**
+ * Constructor.
+ *
+ * @param logRoot worker log root directory
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer) {
this.logRoot = logRoot;
this.resourceAuthorizer = resourceAuthorizer;
}
+ /**
+ * Enumerate dump (profile) files for given worker.
+ *
+ * @param topologyId topology ID
+ * @param hostPort host and port of worker
+ * @param user username
+ * @return The HTML page representing list page of dump files
+ */
public Response listDumpFiles(String topologyId, String hostPort, String user) throws IOException {
String portStr = hostPort.split(":")[1];
File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
@@ -68,6 +83,33 @@ public class LogviewerProfileHandler {
}
}
+ /**
+ * Download a dump file.
+ *
+ * @param topologyId topology ID
+ * @param hostPort host and port of worker
+ * @param fileName dump file name
+ * @param user username
+ * @return a Response which lets browsers download that file.
+ * @see {@link org.apache.storm.daemon.logviewer.utils.LogFileDownloader#downloadFile(String, String, boolean)}
+ */
+ public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException {
+ String portStr = hostPort.split(":")[1];
+ File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
+ File file = new File(dir, fileName);
+
+ if (dir.exists() && file.exists()) {
+ String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
+ if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
+ return LogviewerResponseBuilder.buildDownloadFile(file);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
private String buildDumpFileListPage(String topologyId, String hostPort, File dir) throws IOException {
List<DomContent> liTags = getProfilerDumpFiles(dir).stream()
.map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + file)))
@@ -86,23 +128,6 @@ public class LogviewerProfileHandler {
).render();
}
- public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException {
- String portStr = hostPort.split(":")[1];
- File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
- File file = new File(dir, fileName);
-
- if (dir.exists() && file.exists()) {
- String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
- if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
- return LogviewerResponseBuilder.buildDownloadFile(file);
- } else {
- return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
- }
- } else {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- }
- }
-
private List<String> getProfilerDumpFiles(File dir) throws IOException {
List<File> filesForDir = DirectoryCleaner.getFilesForDir(dir);
return filesForDir.stream().filter(file -> {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
index c8a4f69..7bbb8d7 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java
@@ -15,19 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.logviewer.utils;
-import java.io.IOException;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.DirectoryStream;
-import java.util.Stack;
import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
import java.util.Comparator;
+import java.util.List;
import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.Stack;
import java.util.regex.Pattern;
import org.slf4j.Logger;
@@ -37,25 +38,35 @@ import org.slf4j.LoggerFactory;
* Provide methods to help Logviewer to clean up
* files in directories and to get a list of files without
* worrying about excessive memory usage.
- *
*/
public class DirectoryCleaner {
private static final Logger LOG = LoggerFactory.getLogger(DirectoryCleaner.class);
// used to recognize the pattern of active log files, we may remove the "current" from this list
private static final Pattern ACTIVE_LOG_PATTERN = Pattern.compile(".*\\.(log|err|out|current|yaml|pid)$");
// used to recognize the pattern of some meta files in a worker log directory
- private static final Pattern META_LOG_PATTERN= Pattern.compile(".*\\.(yaml|pid)$");
+ private static final Pattern META_LOG_PATTERN = Pattern.compile(".*\\.(yaml|pid)$");// max number of files to delete for every round
+
+ private static final int PQ_SIZE = 1024;
+ private static final int MAX_ROUNDS = 512; // max rounds of scanning the dirs
+ public static final int MAX_NUMBER_OF_FILES_FOR_DIR = 1024;
// not defining this as static is to allow for mocking in tests
+
+ /**
+ * Creates DirectoryStream for give directory.
+ *
+ * @param dir File instance representing specific directory
+ * @return DirectoryStream
+ */
public DirectoryStream<Path> getStreamForDirectory(File dir) throws IOException {
- DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath());
- return stream;
+ return Files.newDirectoryStream(dir.toPath());
}
/**
* If totalSize of files exceeds the either the per-worker quota or global quota,
* Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs.
* We use the parameter forPerDir to switch between the two deletion modes.
+ *
* @param dirs the list of directories to be scanned for deletion
* @param quota the per-dir quota or the total quota for the all directories
* @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories globally
@@ -64,8 +75,6 @@ public class DirectoryCleaner {
*/
public int deleteOldestWhileTooLarge(List<File> dirs,
long quota, boolean forPerDir, Set<String> activeDirs) throws IOException {
- final int PQ_SIZE = 1024; // max number of files to delete for every round
- final int MAX_ROUNDS = 512; // max rounds of scanning the dirs
long totalSize = 0;
int deletedFiles = 0;
@@ -133,12 +142,12 @@ public class DirectoryCleaner {
round++;
if (round >= MAX_ROUNDS) {
if (forPerDir) {
- LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " +
- "a single directory : {}, will delete the rest files in next interval.",
+ LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in "
+ + "a single directory : {}, will delete the rest files in next interval.",
MAX_ROUNDS, dirs.get(0).getCanonicalPath());
} else {
- LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " +
- "will delete the rest files in next interval.", MAX_ROUNDS);
+ LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, "
+ + "will delete the rest files in next interval.", MAX_ROUNDS);
}
break;
}
@@ -165,15 +174,19 @@ public class DirectoryCleaner {
return false;
}
- // Note that to avoid memory problem, we only return the first 1024 files in a directory
+ /**
+ * Lists files in directory.
+ * Note that to avoid memory problem, we only return the first 1024 files in a directory.
+ *
+ * @param dir directory to get file list
+ * @return files in directory
+ */
public static List<File> getFilesForDir(File dir) throws IOException {
List<File> files = new ArrayList<File>();
- final int MAX_NUM = 1024;
-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath())) {
for (Path path : stream) {
files.add(path.toFile());
- if (files.size() >= MAX_NUM) {
+ if (files.size() >= MAX_NUMBER_OF_FILES_FOR_DIR) {
break;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
index 6e4e073..bfd80a3 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java
@@ -18,16 +18,16 @@
package org.apache.storm.daemon.logviewer.utils;
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
+
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB;
+
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
-import org.apache.storm.StormTimer;
-import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
@@ -41,23 +41,24 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.function.BinaryOperator;
import java.util.stream.StreamSupport;
-import static java.util.stream.Collectors.joining;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toList;
-import static java.util.stream.Collectors.toMap;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB;
+import org.apache.commons.io.IOUtils;
+import org.apache.storm.StormTimer;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * Cleans dead workers logs and directories.
+ */
public class LogCleaner implements Runnable, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
@@ -71,6 +72,14 @@ public class LogCleaner implements Runnable, Closeable {
private final long maxSumWorkerLogsSizeMb;
private long maxPerWorkerLogsSizeMb;
+ /**
+ * Constuctor.
+ *
+ * @param stormConf configuration map for Storm cluster
+ * @param workerLogs {@link WorkerLogs} instance
+ * @param directoryCleaner {@link DirectoryCleaner} instance
+ * @param logRootDir root log directory
+ */
public LogCleaner(Map<String, Object> stormConf, WorkerLogs workerLogs, DirectoryCleaner directoryCleaner,
File logRootDir) {
this.stormConf = stormConf;
@@ -87,6 +96,9 @@ public class LogCleaner implements Runnable, Closeable {
maxSumWorkerLogsSizeMb, maxPerWorkerLogsSizeMb);
}
+ /**
+ * Start log cleanup thread.
+ */
public void start() {
if (intervalSecs != null) {
LOG.debug("starting log cleanup thread at interval: {}", intervalSecs);
@@ -100,9 +112,9 @@ public class LogCleaner implements Runnable, Closeable {
} else {
LOG.warn("The interval for log cleanup is not set. Skip starting log cleanup thread.");
}
-
}
+ @Override
public void close() {
if (logviewerCleanupTimer != null) {
try {
@@ -116,6 +128,7 @@ public class LogCleaner implements Runnable, Closeable {
/**
* Delete old log dirs for which the workers are no longer alive.
*/
+ @Override
public void run() {
try {
int nowSecs = Time.currentTimeSecs();
@@ -152,7 +165,8 @@ public class LogCleaner implements Runnable, Closeable {
@VisibleForTesting
List<Integer> perWorkerDirCleanup(long size) {
return workerLogs.getAllWorkerDirs().stream()
- .map(Unchecked.function(dir -> directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null)))
+ .map(Unchecked.function(dir ->
+ directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null)))
.collect(toList());
}
@@ -167,14 +181,6 @@ public class LogCleaner implements Runnable, Closeable {
return directoryCleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs);
}
- private Set<String> getAliveIds(int nowSecs) throws Exception {
- return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
- .filter(entry -> Objects.nonNull(entry.getValue())
- && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf))
- .map(Map.Entry::getKey)
- .collect(toCollection(TreeSet::new));
- }
-
/**
* Delete the topo dir if it contains zero port dirs.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
index 7d9cb50..62c08da 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
@@ -18,22 +18,38 @@
package org.apache.storm.daemon.logviewer.utils;
-import javax.ws.rs.core.Response;
import java.io.File;
import java.io.IOException;
+import javax.ws.rs.core.Response;
+
public class LogFileDownloader {
private final String logRoot;
private final String daemonLogRoot;
private final ResourceAuthorizer resourceAuthorizer;
+ /**
+ * Constructor.
+ *
+ * @param logRoot root worker log directory
+ * @param daemonLogRoot root daemon log directory
+ * @param resourceAuthorizer {@link ResourceAuthorizer}
+ */
public LogFileDownloader(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
this.logRoot = logRoot;
this.daemonLogRoot = daemonLogRoot;
this.resourceAuthorizer = resourceAuthorizer;
}
+ /**
+ * Checks authorization for the log file and download
+ *
+ * @param fileName file to download
+ * @param user username
+ * @param isDaemon true if the file is a daemon log, false if the file is an worker log
+ * @return a Response which lets browsers download that file.
+ */
public Response downloadFile(String fileName, String user, boolean isDaemon) throws IOException {
String rootDir = isDaemon ? daemonLogRoot : logRoot;
File file = new File(rootDir, fileName).getCanonicalFile();
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
index d0f2857..59570ee 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java
@@ -18,13 +18,12 @@
package org.apache.storm.daemon.logviewer.utils;
-import com.google.common.io.ByteStreams;
-import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.ui.UIHelpers;
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.h2;
+import static javax.ws.rs.core.Response.Status.OK;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
+import com.google.common.io.ByteStreams;
import java.io.BufferedOutputStream;
import java.io.File;
@@ -35,26 +34,46 @@ import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
-import static j2html.TagCreator.body;
-import static j2html.TagCreator.h2;
-import static javax.ws.rs.core.Response.Status.OK;
-import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.ui.UIHelpers;
public class LogviewerResponseBuilder {
private LogviewerResponseBuilder() {
}
+ /**
+ * Build a Response object representing success response with HTML entity.
+ *
+ * @param content HTML entity content, String type
+ */
public static Response buildSuccessHtmlResponse(String content) {
return Response.status(OK).entity(content)
.type(MediaType.TEXT_HTML_TYPE).build();
}
+ /**
+ * Build a Response object representing success response with JSON entity.
+ *
+ * @param entity entity object to represent it as JSON
+ * @param callback callback for JSONP
+ * @param origin origin
+ * @see {@link JsonResponseBuilder}
+ */
public static Response buildSuccessJsonResponse(Object entity, String callback, String origin) {
return new JsonResponseBuilder().setData(entity).setCallback(callback)
.setHeaders(LogviewerResponseBuilder.getHeadersForSuccessResponse(origin)).build();
}
+ /**
+ * Build a Response object representing download a file.
+ *
+ * @param file file to download
+ */
public static Response buildDownloadFile(File file) throws IOException {
// do not close this InputStream in method: it will be used from jetty server
InputStream is = new FileInputStream(file);
@@ -65,6 +84,11 @@ public class LogviewerResponseBuilder {
.build();
}
+ /**
+ * Build a Response object representing unauthorized user, with HTML response.
+ *
+ * @param user username
+ */
public static Response buildResponseUnautohrizedUser(String user) {
String entity = buildUnauthorizedUserHtml(user);
return Response.status(OK)
@@ -73,6 +97,9 @@ public class LogviewerResponseBuilder {
.build();
}
+ /**
+ * Build a Response object representing page not found.
+ */
public static Response buildResponsePageNotFound() {
return Response.status(404)
.entity("Page not found")
@@ -80,11 +107,23 @@ public class LogviewerResponseBuilder {
.build();
}
+ /**
+ * Build a Response object representing unauthorized user, with JSON response.
+ *
+ * @param user username
+ * @param callback callback for JSONP
+ */
public static Response buildUnauthorizedUserJsonResponse(String user, String callback) {
return new JsonResponseBuilder().setData(UIHelpers.unauthorizedUserJson(user))
.setCallback(callback).setStatus(401).build();
}
+ /**
+ * Build a Response object representing exception, with JSON response.
+ *
+ * @param ex Exception object
+ * @param callback callback for JSONP
+ */
public static Response buildExceptionJsonResponse(Exception ex, String callback) {
return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(ex))
.setCallback(callback).setStatus(500).build();
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
index 55691b2..16f408a 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
@@ -18,16 +18,10 @@
package org.apache.storm.daemon.logviewer.utils;
+import static org.apache.storm.DaemonConfig.UI_FILTER;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.Config;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.security.auth.AuthUtils;
-import org.apache.storm.security.auth.IGroupMappingServiceProvider;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ServerConfigUtils;
-import org.apache.storm.utils.Utils;
import java.io.File;
import java.io.IOException;
@@ -37,28 +31,51 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import static org.apache.storm.DaemonConfig.UI_FILTER;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.security.auth.AuthUtils;
+import org.apache.storm.security.auth.IGroupMappingServiceProvider;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerConfigUtils;
+import org.apache.storm.utils.Utils;
public class ResourceAuthorizer {
private final Map<String, Object> stormConf;
private final IGroupMappingServiceProvider groupMappingServiceProvider;
+ /**
+ * Constuctor.
+ *
+ * @param stormConf storm configuration
+ */
public ResourceAuthorizer(Map<String, Object> stormConf) {
this.stormConf = stormConf;
this.groupMappingServiceProvider = AuthUtils.GetGroupMappingServiceProviderPlugin(stormConf);
}
+ /**
+ * Checks whether user is allowed to access file via UI. Always true when UI filter is not set.
+ *
+ * @param fileName file name to access
+ * @param user username
+ */
public boolean isUserAllowedToAccessFile(String fileName, String user) {
return isUiFilterNotSet() || isAuthorizedLogUser(user, fileName);
}
+ /**
+ * Checks whether user is authorized to access file. Checks regardless of UI filter.
+ *
+ * @param user username
+ * @param fileName file name to access
+ */
public boolean isAuthorizedLogUser(String user, String fileName) {
if (StringUtils.isEmpty(user) || StringUtils.isEmpty(fileName)
|| getLogUserGroupWhitelist(fileName) == null) {
return false;
} else {
- Set<String> groups = getUserGroups(user);
LogUserGroupWhitelist whitelist = getLogUserGroupWhitelist(fileName);
List<String> logsUsers = new ArrayList<>();
@@ -70,11 +87,18 @@ public class ResourceAuthorizer {
logsGroups.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_GROUPS)));
logsGroups.addAll(whitelist.getGroupWhitelist());
- return logsUsers.stream().anyMatch(u -> u.equals(user)) ||
- Sets.intersection(groups, new HashSet<>(logsGroups)).size() > 0;
+ Set<String> groups = getUserGroups(user);
+
+ return logsUsers.stream().anyMatch(u -> u.equals(user))
+ || Sets.intersection(groups, new HashSet<>(logsGroups)).size() > 0;
}
}
+ /**
+ * Get the whitelist of users and groups for given file.
+ *
+ * @param fileName file name to get the whitelist
+ */
public LogUserGroupWhitelist getLogUserGroupWhitelist(String fileName) {
File wlFile = ServerConfigUtils.getLogMetaDataFile(fileName);
Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(wlFile.getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
index 6018b79..eba6404 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java
@@ -18,16 +18,13 @@
package org.apache.storm.daemon.logviewer.utils;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.storm.Config.SUPERVISOR_RUN_WORKER_AS_USER;
+import static org.apache.storm.Config.TOPOLOGY_SUBMITTER_USER;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+
import com.google.common.collect.Lists;
-import org.apache.storm.daemon.supervisor.ClientSupervisorUtils;
-import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
-import org.jooq.lambda.tuple.Tuple2;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -43,12 +40,19 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.stream.Stream;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toMap;
-import static org.apache.storm.Config.SUPERVISOR_RUN_WORKER_AS_USER;
-import static org.apache.storm.Config.TOPOLOGY_SUBMITTER_USER;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+import org.apache.storm.daemon.supervisor.ClientSupervisorUtils;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+import org.jooq.lambda.tuple.Tuple2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+/**
+ * A class that knows about how to operate with worker log directory.
+ */
public class WorkerLogs {
private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
@@ -56,19 +60,30 @@ public class WorkerLogs {
private final Map<String, Object> stormConf;
private final File logRootDir;
+ /**
+ * Constructor.
+ *
+ * @param stormConf storm configuration
+ * @param logRootDir the log root directory
+ */
public WorkerLogs(Map<String, Object> stormConf, File logRootDir) {
this.stormConf = stormConf;
this.logRootDir = logRootDir;
}
+ /**
+ * Set permission of log file so that logviewer can serve the file.
+ *
+ * @param fileName log file
+ */
public void setLogFilePermission(String fileName) throws IOException {
File file = new File(logRootDir, fileName).getCanonicalFile();
boolean runAsUser = ObjectReader.getBoolean(stormConf.get(SUPERVISOR_RUN_WORKER_AS_USER), false);
File parent = new File(logRootDir, fileName).getParentFile();
Optional<File> mdFile = (parent == null) ? Optional.empty() : getMetadataFileForWorkerLogDir(parent);
- Optional<String> topoOwner = mdFile.isPresent() ?
- Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().getCanonicalPath())) :
- Optional.empty();
+ Optional<String> topoOwner = mdFile.isPresent()
+ ? Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().getCanonicalPath()))
+ : Optional.empty();
if (runAsUser && topoOwner.isPresent() && file.exists() && !Files.isReadable(file.toPath())) {
LOG.debug("Setting permissions on file {} with topo-owner {}", fileName, topoOwner);
@@ -78,6 +93,9 @@ public class WorkerLogs {
}
}
+ /**
+ * Return a list of all log files from worker directories in root log directory.
+ */
public List<File> getAllLogsForRootDir() throws IOException {
List<File> files = new ArrayList<>();
Set<File> topoDirFiles = getAllWorkerDirs();
@@ -90,6 +108,9 @@ public class WorkerLogs {
return files;
}
+ /**
+ * Return a set of all worker directories in root log directory.
+ */
public Set<File> getAllWorkerDirs() {
File[] rootDirFiles = logRootDir.listFiles();
if (rootDirFiles != null) {
@@ -116,6 +137,10 @@ public class WorkerLogs {
.collect(toCollection(TreeSet::new));
}
+ /**
+ * Return a metadata file (worker.yaml) for given worker log directory.
+ * @param logDir worker log directory
+ */
public Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
File metaFile = new File(logDir, WORKER_YAML);
if (metaFile.exists()) {
@@ -126,16 +151,31 @@ public class WorkerLogs {
}
}
+ /**
+ * Return worker id from worker meta file.
+ *
+ * @param metaFile metadata file
+ */
public String getWorkerIdFromMetadataFile(String metaFile) {
Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
return ObjectReader.getString(map.get("worker-id"), null);
}
+ /**
+ * Return topology owner from worker meta file.
+ *
+ * @param metaFile metadata file
+ */
public String getTopologyOwnerFromMetadataFile(String metaFile) {
Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
return ObjectReader.getString(map.get(TOPOLOGY_SUBMITTER_USER), null);
}
+ /**
+ * Retrieve the set of alive worker IDs.
+ *
+ * @param nowSecs current time in seconds
+ */
public Set<String> getAliveIds(int nowSecs) throws Exception {
return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
.filter(entry -> Objects.nonNull(entry.getValue())
@@ -144,6 +184,12 @@ public class WorkerLogs {
.collect(toCollection(TreeSet::new));
}
+ /**
+ * Finds a worker ID for each directory in set and return it as map.
+ *
+ * @param logDirs directories to check whether they're worker directories or not
+ * @return the pair of worker ID, directory. worker ID will be an empty string if the directory is not a worker directory.
+ */
public Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
// we could also make this static, but not to do it due to mock
return logDirs.stream().map(Unchecked.function(logDir -> {
@@ -156,6 +202,8 @@ public class WorkerLogs {
/**
* Return the path of the worker log with the format of topoId/port/worker.log.*
+ *
+ * @param file worker log
*/
public static String getTopologyPortWorkerLog(File file) {
try {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
index 32a723e..c44e8cd 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java
@@ -18,31 +18,32 @@
package org.apache.storm.daemon.logviewer.webapp;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME;
+
+import java.io.File;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.ApplicationPath;
+import javax.ws.rs.core.Application;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.Appender;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.RollingFileAppender;
import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
-import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
import org.apache.storm.security.auth.AuthUtils;
import org.apache.storm.security.auth.IHttpCredentialsPlugin;
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.ObjectReader;
-import javax.ws.rs.ApplicationPath;
-import javax.ws.rs.core.Application;
-import java.io.File;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME;
-
@ApplicationPath("")
public class LogviewerApplication extends Application {
private static Map<String, Object> stormConf;
@@ -75,6 +76,11 @@ public class LogviewerApplication extends Application {
return singletons;
}
+ /**
+ * Spot to inject storm configuration before initializing LogviewerApplication instance.
+ *
+ * @param stormConf storm configuration
+ */
public static void setup(Map<String, Object> stormConf) {
LogviewerApplication.stormConf = stormConf;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
index bc6529c..478001d 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java
@@ -19,13 +19,25 @@
package org.apache.storm.daemon.logviewer.webapp;
import com.codahale.metrics.Meter;
+
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
-import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogDownloadHandler;
import org.apache.storm.daemon.logviewer.handler.LogviewerLogPageHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerLogSearchHandler;
+import org.apache.storm.daemon.logviewer.handler.LogviewerProfileHandler;
import org.apache.storm.metric.StormMetricsRegistry;
import org.apache.storm.security.auth.IHttpCredentialsPlugin;
import org.apache.storm.ui.InvalidRequestException;
@@ -33,24 +45,20 @@ import org.apache.storm.ui.UIHelpers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.net.URLDecoder;
-import java.util.Map;
-
+/**
+ * Handles HTTP requests for Logviewer.
+ */
@Path("/")
public class LogviewerResource {
private static final Logger LOG = LoggerFactory.getLogger(LogviewerResource.class);
private static final Meter meterLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-log-page-http-requests");
- private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-daemonlog-page-http-requests");
- private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-file-http-requests");
- private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-daemon-file-http-requests");
+ private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter(
+ "logviewer:num-daemonlog-page-http-requests");
+ private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter(
+ "logviewer:num-download-log-file-http-requests");
+ private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter(
+ "logviewer:num-download-log-daemon-file-http-requests");
private static final Meter meterListLogsHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-list-logs-http-requests");
private final LogviewerLogPageHandler logviewer;
@@ -59,6 +67,15 @@ public class LogviewerResource {
private final LogviewerLogSearchHandler logSearchHandler;
private final IHttpCredentialsPlugin httpCredsHandler;
+ /**
+ * Constructor.
+ *
+ * @param logviewerParam {@link LogviewerLogPageHandler}
+ * @param profileHandler {@link LogviewerProfileHandler}
+ * @param logDownloadHandler {@link LogviewerLogDownloadHandler}
+ * @param logSearchHandler {@link LogviewerLogSearchHandler}
+ * @param httpCredsHandler {@link IHttpCredentialsPlugin}
+ */
public LogviewerResource(LogviewerLogPageHandler logviewerParam, LogviewerProfileHandler profileHandler,
LogviewerLogDownloadHandler logDownloadHandler, LogviewerLogSearchHandler logSearchHandler,
IHttpCredentialsPlugin httpCredsHandler) {
@@ -69,6 +86,9 @@ public class LogviewerResource {
this.httpCredsHandler = httpCredsHandler;
}
+ /**
+ * Handles '/log' request.
+ */
@GET
@Path("/log")
public Response log(@Context HttpServletRequest request) throws IOException {
@@ -87,6 +107,9 @@ public class LogviewerResource {
}
}
+ /**
+ * Handles '/daemonlog' request.
+ */
@GET
@Path("/daemonlog")
public Response daemonLog(@Context HttpServletRequest request) throws IOException {
@@ -105,6 +128,9 @@ public class LogviewerResource {
}
}
+ /**
+ * Handles '/searchLogs' request.
+ */
@GET
@Path("/searchLogs")
public Response searchLogs(@Context HttpServletRequest request) throws IOException {
@@ -117,6 +143,9 @@ public class LogviewerResource {
return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
}
+ /**
+ * Handles '/listLogs' request.
+ */
@GET
@Path("/listLogs")
public Response listLogs(@Context HttpServletRequest request) throws IOException {
@@ -131,6 +160,9 @@ public class LogviewerResource {
return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
}
+ /**
+ * Handles '/dumps' (listing dump files) request.
+ */
@GET
@Path("/dumps/{topo-id}/{host-port}")
public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
@@ -139,6 +171,9 @@ public class LogviewerResource {
return profileHandler.listDumpFiles(topologyId, hostPort, user);
}
+ /**
+ * Handles '/dumps' (downloading specific dump file) request.
+ */
@GET
@Path("/dumps/{topo-id}/{host-port}/{filename}")
public Response downloadDumpFile(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
@@ -147,6 +182,9 @@ public class LogviewerResource {
return profileHandler.downloadDumpFile(topologyId, hostPort, fileName, user);
}
+ /**
+ * Handles '/download' (downloading specific log file) request.
+ */
@GET
@Path("/download")
public Response downloadLogFile(@Context HttpServletRequest request) throws IOException {
@@ -158,6 +196,9 @@ public class LogviewerResource {
return logDownloadHandler.downloadLogFile(decodedFileName, user);
}
+ /**
+ * Handles '/daemondownload' (downloading specific daemon log file) request.
+ */
@GET
@Path("/daemondownload")
public Response downloadDaemonLogFile(@Context HttpServletRequest request) throws IOException {
@@ -169,6 +210,9 @@ public class LogviewerResource {
return logDownloadHandler.downloadDaemonLogFile(decodedFileName, user);
}
+ /**
+ * Handles '/search' (searching from specific worker or daemon log file) request.
+ */
@GET
@Path("/search/{file}")
public Response search(@PathParam("file") String file, @Context HttpServletRequest request) throws IOException {
@@ -191,6 +235,9 @@ public class LogviewerResource {
}
}
+ /**
+ * Handles '/deepSearch' request.
+ */
@GET
@Path("/deepSearch/{topoId}")
public Response deepSearch(@PathParam("topoId") String topologyId,
[12/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java
index 83b3588..700fbb7 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java
@@ -21,7 +21,17 @@ package org.apache.storm.daemon.utils;
import java.util.List;
import java.util.stream.Collectors;
+/**
+ * Implements missing sequence functions in Java compared to Clojure.
+ * To make thing simpler, it only supports List type.
+ */
public class ListFunctionalSupport {
+ /**
+ * Get the first element in list.
+ *
+ * @param list list to get
+ * @return the first element. null if list is null or empty.
+ */
public static <T> T first(List<T> list) {
if (list == null || list.size() <= 0) {
return null;
@@ -30,6 +40,28 @@ public class ListFunctionalSupport {
return list.get(0);
}
+ /**
+ * get the last element in list.
+ *
+ * @param list list to get
+ * @return the last element. null if list is null or empty.
+ */
+ public static <T> T last(List<T> list) {
+ if (list == null || list.size() <= 0) {
+ return null;
+ }
+
+ return list.get(list.size() - 1);
+ }
+
+ /**
+ * Get the last N elements as a new list.
+ *
+ * @param list list to get
+ * @param count element count to get
+ * @return the first element. null if list is null.
+ * elements in a new list may be less than count if there're not enough elements in the list.
+ */
public static <T> List<T> takeLast(List<T> list, int count) {
if (list == null) {
return null;
@@ -45,6 +77,13 @@ public class ListFunctionalSupport {
}
}
+ /**
+ * Drop the first N elements and create a new list.
+ *
+ * @param list the list
+ * @param count element count to drop
+ * @return newly created sublist that drops the first N elements from origin list. null if list is null.
+ */
public static <T> List<T> drop(List<T> list, int count) {
if (list == null) {
return null;
@@ -55,15 +94,15 @@ public class ListFunctionalSupport {
.collect(Collectors.toList());
}
+ /**
+ * Drop the only first element and create a new list. equivalent to drop(list, 1).
+ *
+ * @see {@link ListFunctionalSupport#drop(List, int)}
+ * @param list the list
+ * @return newly created sublist that drops the first element from origin list. null if list is null.
+ */
public static <T> List<T> rest(List<T> list) {
return drop(list, 1);
}
- public static <T> T last(List<T> list) {
- if (list == null || list.size() <= 0) {
- return null;
- }
-
- return list.get(list.size() - 1);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java
index 24c630d..8e8b5d0 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java
@@ -21,12 +21,21 @@ package org.apache.storm.daemon.utils;
import java.io.IOException;
import java.io.InputStream;
+/**
+ * Utility class for (Input/Output)Stream.
+ */
public class StreamUtil {
private StreamUtil() {
}
/**
+ * Skips over and discards N bytes of data from the input stream.
+ * <p/>
* FileInputStream#skip may not work the first time, so ensure it successfully skips the given number of bytes.
+ *
+ * @see {@link java.io.FileInputStream#skip(long)}
+ * @param stream the stream to skip
+ * @param n bytes to skip
*/
public static void skipBytes(InputStream stream, Integer n) throws IOException {
long skipped = 0;
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java
deleted file mode 100644
index d30ac65..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.utils;
-
-import java.net.URLEncoder;
-import java.util.Map;
-
-import static java.util.stream.Collectors.joining;
-
-public class URLBuilder {
- private URLBuilder() {
- }
-
- public static String build(String urlPath, Map<String, Object> parameters) {
- StringBuilder sb = new StringBuilder();
- sb.append(urlPath);
- if (parameters.size() > 0) {
- sb.append("?");
-
- String queryParam = parameters.entrySet().stream()
- .map(entry -> URLEncoder.encode(entry.getKey()) + "=" + URLEncoder.encode(entry.getValue().toString()))
- .collect(joining("&"));
- sb.append(queryParam);
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/main/java/org/apache/storm/daemon/utils/UrlBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/UrlBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/UrlBuilder.java
new file mode 100644
index 0000000..cea6dcc
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/UrlBuilder.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.utils;
+
+import static java.util.stream.Collectors.joining;
+
+import java.net.URLEncoder;
+import java.util.Map;
+
+/**
+ * Convenient utility class to build the URL.
+ */
+public class UrlBuilder {
+ private UrlBuilder() {
+ }
+
+ /**
+ * Build the URL. the key and value of query parameters will be encoded.
+ *
+ * @param urlPath URL except query parameter
+ * @param parameters query parameter
+ * @return the url concatenating url path and query parameters
+ */
+ public static String build(String urlPath, Map<String, Object> parameters) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(urlPath);
+ if (parameters.size() > 0) {
+ sb.append("?");
+
+ String queryParam = parameters.entrySet().stream()
+ .map(entry -> URLEncoder.encode(entry.getKey()) + "=" + URLEncoder.encode(entry.getValue().toString()))
+ .collect(joining("&"));
+ sb.append(queryParam);
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java
index 9ab623f..a8452a9 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.drpc;
import static org.junit.Assert.assertEquals;
@@ -90,8 +91,8 @@ public class DRPCServerTest {
try (DRPCServer server = new DRPCServer(conf)) {
server.start();
try (DRPCClient client = new DRPCClient(conf, "localhost", server.getDrpcPort());
- DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) {
- Future<String> found = exec.submit(() -> client.getClient().execute("testing", "test"));
+ DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) {
+ final Future<String> found = exec.submit(() -> client.getClient().execute("testing", "test"));
DRPCRequest request = getNextAvailableRequest(invoke, "testing");
assertNotNull(request);
assertEquals("test", request.get_func_args());
@@ -129,9 +130,9 @@ public class DRPCServerTest {
}
}
- public static String GET(int port, String func, String args) {
+ private static String doGet(int port, String func, String args) {
try {
- URL url = new URL("http://localhost:"+port+"/drpc/"+func+"/"+args);
+ URL url = new URL("http://localhost:" + port + "/drpc/" + func + "/" + args);
InputStream in = url.openStream();
byte[] buffer = new byte[1024];
int read = in.read(buffer);
@@ -150,7 +151,7 @@ public class DRPCServerTest {
//TODO need a better way to do this
Thread.sleep(2000);
try (DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) {
- Future<String> found = exec.submit(() -> GET(server.getHttpServerPort(), "testing", "test"));
+ final Future<String> found = exec.submit(() -> doGet(server.getHttpServerPort(), "testing", "test"));
DRPCRequest request = getNextAvailableRequest(invoke, "testing");
assertNotNull(request);
assertEquals("test", request.get_func_args());
@@ -171,7 +172,7 @@ public class DRPCServerTest {
//TODO need a better way to do this
Thread.sleep(2000);
try (DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) {
- Future<String> found = exec.submit(() -> GET(server.getHttpServerPort(), "testing", "test"));
+ Future<String> found = exec.submit(() -> doGet(server.getHttpServerPort(), "testing", "test"));
DRPCRequest request = getNextAvailableRequest(invoke, "testing");
assertNotNull(request);
assertEquals("test", request.get_func_args());
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java
deleted file mode 100644
index 6abe153..0000000
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.logviewer;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Path;
-import java.util.Iterator;
-import java.util.List;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class LogviewerTest {
- private Path mkMockPath(File file) {
- Path mockPath = mock(Path.class);
- when(mockPath.toFile()).thenReturn(file);
- return mockPath;
- }
-
- private DirectoryStream<Path> mkDirectoryStream(List<Path> listOfPaths) {
- return new DirectoryStream<Path>() {
- @Override
- public Iterator<Path> iterator() {
- return listOfPaths.iterator();
- }
-
- @Override
- public void close() throws IOException {
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
index 2c05972..58daa30 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
@@ -18,15 +18,11 @@
package org.apache.storm.daemon.logviewer.handler;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
-import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
-import org.apache.storm.utils.Utils;
-import org.assertj.core.util.Lists;
-import org.junit.Test;
-import javax.ws.rs.core.Response;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -34,8 +30,14 @@ import java.nio.file.attribute.FileAttribute;
import java.util.List;
import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import javax.ws.rs.core.Response;
+
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.logviewer.utils.WorkerLogs;
+import org.apache.storm.utils.Utils;
+import org.assertj.core.util.Lists;
+import org.junit.Test;
public class LogviewerLogPageHandlerTest {
@@ -62,27 +64,27 @@ public class LogviewerLogPageHandlerTest {
LogviewerLogPageHandler handler = new LogviewerLogPageHandler(rootPath, null,
new WorkerLogs(stormConf, new File(rootPath)), new ResourceAuthorizer(stormConf));
- Response expectedAll = LogviewerResponseBuilder.buildSuccessJsonResponse(
+ final Response expectedAll = LogviewerResponseBuilder.buildSuccessJsonResponse(
Lists.newArrayList("topoA/port1/worker.log", "topoA/port2/worker.log", "topoB/port1/worker.log"),
null,
origin
);
- Response expectedFilterPort = LogviewerResponseBuilder.buildSuccessJsonResponse(
+ final Response expectedFilterPort = LogviewerResponseBuilder.buildSuccessJsonResponse(
Lists.newArrayList("topoA/port1/worker.log", "topoB/port1/worker.log"),
null,
origin
);
- Response expectedFilterTopoId = LogviewerResponseBuilder.buildSuccessJsonResponse(
+ final Response expectedFilterTopoId = LogviewerResponseBuilder.buildSuccessJsonResponse(
Lists.newArrayList("topoB/port1/worker.log"),
null,
origin
);
- Response returnedAll = handler.listLogFiles("user", null, null, null, origin);
- Response returnedFilterPort = handler.listLogFiles("user", 1111, null, null, origin);
- Response returnedFilterTopoId = handler.listLogFiles("user", null, "topoB", null, origin);
+ final Response returnedAll = handler.listLogFiles("user", null, null, null, origin);
+ final Response returnedFilterPort = handler.listLogFiles("user", 1111, null, null, origin);
+ final Response returnedFilterTopoId = handler.listLogFiles("user", null, "topoB", null, origin);
Utils.forceDelete(rootPath);
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
index 9e36b5a..c7f927b 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
@@ -18,6 +18,33 @@
package org.apache.storm.daemon.logviewer.handler;
+import static java.util.stream.Collectors.joining;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.internal.verification.VerificationModeFactory.times;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.attribute.FileAttribute;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
import org.apache.commons.lang.StringUtils;
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.logviewer.LogviewerConstant;
@@ -34,33 +61,6 @@ import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
-import java.io.File;
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.nio.file.Files;
-import java.nio.file.attribute.FileAttribute;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-
-import static java.util.stream.Collectors.joining;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.mockito.internal.verification.VerificationModeFactory.times;
-
@RunWith(Enclosed.class)
public class LogviewerLogSearchHandlerTest {
@@ -98,8 +98,8 @@ public class LogviewerLogSearchHandlerTest {
String actualUrl = handler.urlToMatchCenteredInLogPage(new byte[42], expectedFname, 27526, 8888);
- assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/log?file=" + expectedFname +
- "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl);
+ assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/log?file=" + expectedFname
+ + "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl);
} finally {
Utils.setInstance(prevUtils);
}
@@ -119,13 +119,14 @@ public class LogviewerLogSearchHandlerTest {
String actualUrl = handler.urlToMatchCenteredInLogPageDaemonFile(new byte[42], expectedFname, 27526, 8888);
- assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/daemonlog?file=" + expectedFname +
- "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl);
+ assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/daemonlog?file=" + expectedFname
+ + "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl);
} finally {
Utils.setInstance(prevUtils);
}
}
+ @SuppressWarnings("checkstyle:LineLength")
@Test
public void testReturnsCorrectBeforeAndAfterContext() throws InvalidRequestException, UnknownHostException {
Utils prevUtils = null;
@@ -135,12 +136,9 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"logviewer-search-context-tests.log.test");
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
- Map<String, Object> searchResult = handler.substringSearch(file, pattern);
-
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
expected.put("searchString", pattern);
@@ -176,6 +174,9 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
@@ -191,12 +192,9 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"small-worker.log.test");
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
- Map<String, Object> searchResult = handler.substringSearch(file, pattern);
-
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
expected.put("searchString", pattern);
@@ -212,6 +210,9 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
@@ -227,12 +228,9 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"small-worker.log.test");
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
- Map<String, Object> searchResult = handler.substringSearchDaemonLog(file, pattern);
-
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "yes");
expected.put("searchString", pattern);
@@ -248,6 +246,9 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearchDaemonLog(file, pattern);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
@@ -263,13 +264,9 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"test-3072.log.test");
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
- Map<String, Object> searchResult = handler.substringSearch(file, pattern);
- Map<String, Object> searchResult2 = handler.substringSearch(file, pattern, 1);
-
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
expected.put("searchString", pattern);
@@ -286,6 +283,10 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+ Map<String, Object> searchResult2 = handler.substringSearch(file, pattern, 1);
+
assertEquals(expected, searchResult);
assertEquals(expected, searchResult2);
} finally {
@@ -293,6 +294,7 @@ public class LogviewerLogSearchHandlerTest {
}
}
+ @SuppressWarnings("checkstyle:LineLength")
@Test
public void testNextByteOffsetsAreCorrectForEachMatch() throws UnknownHostException, InvalidRequestException {
Utils prevUtils = null;
@@ -329,8 +331,6 @@ public class LogviewerLogSearchHandlerTest {
assertEquals(data.v2().intValue(), ((List) result.get("matches")).size());
}));
- Map<String, Object> searchResult = handler.substringSearch(file, pattern, 7);
-
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
expected.put("searchString", pattern);
@@ -390,12 +390,15 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 7);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
}
}
+ @SuppressWarnings("checkstyle:LineLength")
@Test
public void testCorrectMatchOffsetIsReturnedWhenSkippingBytes() throws InvalidRequestException, UnknownHostException {
Utils prevUtils = null;
@@ -405,13 +408,10 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"test-worker.log.test");
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
-
int startByteOffset = 3197;
- Map<String, Object> searchResult = handler.substringSearch(file, pattern, 1, startByteOffset);
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
@@ -430,12 +430,16 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 1, startByteOffset);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
}
}
+ @SuppressWarnings("checkstyle:LineLength")
@Test
public void testAnotherPatterns1() throws UnknownHostException, InvalidRequestException {
Utils prevUtils = null;
@@ -445,13 +449,10 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"test-worker.log.test");
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
-
String pattern = Seq.range(0, 1024).map(x -> "X").collect(joining());
- Map<String, Object> searchResult = handler.substringSearch(file, pattern, 2);
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
@@ -477,12 +478,16 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 2);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
}
}
+ @SuppressWarnings("checkstyle:LineLength")
@Test
public void testAnotherPatterns2() throws UnknownHostException, InvalidRequestException {
Utils prevUtils = null;
@@ -492,13 +497,9 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"test-worker.log.test");
-
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
-
String pattern = "𐄀𐄁𐄂";
- Map<String, Object> searchResult = handler.substringSearch(file, pattern, 1);
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
@@ -517,6 +518,9 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", matches);
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 1);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
@@ -534,13 +538,9 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- File file = new File(String.join(File.separator, "src", "test", "resources"),
+ final File file = new File(String.join(File.separator, "src", "test", "resources"),
"test-worker.log.test");
- LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
-
- Map<String, Object> searchResult = handler.substringSearch(file, pattern);
-
Map<String, Object> expected = new HashMap<>();
expected.put("isDaemon", "no");
expected.put("searchString", pattern);
@@ -548,6 +548,9 @@ public class LogviewerLogSearchHandlerTest {
expected.put("matches", Collections.emptyList());
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+
assertEquals(expected, searchResult);
} finally {
Utils.setInstance(prevUtils);
@@ -578,11 +581,11 @@ public class LogviewerLogSearchHandlerTest {
files.add(new File(String.join(File.separator, "src", "test", "resources"),
"logviewer-search-context-tests.log.gz"));
- LogviewerLogSearchHandler handler = getSearchHandler();
+ final LogviewerLogSearchHandler handler = getSearchHandler();
- List<Map<String, Object>> matches1 = handler.findNMatches(files, 20, 0, 0, "needle").getMatches();
- List<Map<String, Object>> matches2 = handler.findNMatches(files, 20, 0, 126, "needle").getMatches();
- List<Map<String, Object>> matches3 = handler.findNMatches(files, 20, 1, 0, "needle").getMatches();
+ final List<Map<String, Object>> matches1 = handler.findNMatches(files, 20, 0, 0, "needle").getMatches();
+ final List<Map<String, Object>> matches2 = handler.findNMatches(files, 20, 0, 126, "needle").getMatches();
+ final List<Map<String, Object>> matches3 = handler.findNMatches(files, 20, 1, 0, "needle").getMatches();
assertEquals(2, matches1.size());
assertEquals(4, ((List) matches1.get(0).get("matches")).size());
@@ -603,6 +606,9 @@ public class LogviewerLogSearchHandlerTest {
private List<File> logFiles;
private String topoPath;
+ /**
+ * Setup test environment for each test.
+ */
@Before
public void setUp() throws IOException {
logFiles = new ArrayList<>();
@@ -619,6 +625,9 @@ public class LogviewerLogSearchHandlerTest {
new File(topoPath, "6700").createNewFile();
}
+ /**
+ * Clean up test environment.
+ */
@After
public void tearDown() {
if (StringUtils.isNotEmpty(topoPath)) {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java
index e52c9db..7fc0ab6 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java
@@ -18,13 +18,22 @@
package org.apache.storm.daemon.logviewer.testsupport;
-import org.mockito.ArgumentCaptor;
+import static org.junit.Assert.assertEquals;
import java.util.function.Consumer;
+import org.mockito.ArgumentCaptor;
-import static org.junit.Assert.assertEquals;
-
+/**
+ * Help verifying captured argument in Mockito.
+ */
public class ArgumentsVerifier {
+ /**
+ * Asserting that method is called with expected first argument.
+ *
+ * @param verifyConsumer Consumer implementation that takes ArgumentCaptor and call 'Mockito.verify'
+ * @param argClazz Class type for argument
+ * @param expectedArg expected argument
+ */
public static <T> void verifyFirstCallArgsForSingleArgMethod(Consumer<ArgumentCaptor<T>> verifyConsumer,
Class<T> argClazz, T expectedArg) {
ArgumentCaptor<T> captor = ArgumentCaptor.forClass(argClazz);
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java
index 7215456..b8be8ad 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java
@@ -18,17 +18,20 @@
package org.apache.storm.daemon.logviewer.testsupport;
-import java.io.File;
-import java.io.IOException;
-
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.IOException;
+
public class MockDirectoryBuilder {
private String dirName;
private long mtime;
private File[] files;
+ /**
+ * Constructor.
+ */
public MockDirectoryBuilder() {
this.dirName = "adir";
this.mtime = 1;
@@ -50,6 +53,9 @@ public class MockDirectoryBuilder {
return this;
}
+ /**
+ * Build mocked File object with given (or default) attributes for a directory.
+ */
public File build() {
File mockFile = mock(File.class);
when(mockFile.getName()).thenReturn(dirName);
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java
index 385eec4..e6a7c20 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java
@@ -18,17 +18,20 @@
package org.apache.storm.daemon.logviewer.testsupport;
-import java.io.File;
-import java.io.IOException;
-
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.IOException;
+
public class MockFileBuilder {
private String fileName;
private long mtime;
private long length;
+ /**
+ * Constructor.
+ */
public MockFileBuilder() {
this.fileName = "afile";
this.mtime = 1;
@@ -50,6 +53,9 @@ public class MockFileBuilder {
return this;
}
+ /**
+ * Build mocked File object with given (or default) attributes for a file.
+ */
public File build() {
File mockFile = mock(File.class);
when(mockFile.getName()).thenReturn(fileName);
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
index 21f4e9e..1031c04 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
@@ -18,15 +18,22 @@
package org.apache.storm.daemon.logviewer.utils;
-import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.daemon.logviewer.testsupport.MockDirectoryBuilder;
-import org.apache.storm.daemon.logviewer.testsupport.MockFileBuilder;
-import org.apache.storm.generated.LSWorkerHeartbeat;
-import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Seq;
-import org.junit.Test;
-import org.mockito.internal.util.collections.Sets;
+import static java.util.stream.Collectors.toList;
+import static org.apache.storm.Config.SUPERVISOR_WORKER_TIMEOUT_SECS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyMapOf;
+import static org.mockito.Matchers.anySetOf;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileFilter;
@@ -40,27 +47,19 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-import static java.util.stream.Collectors.toList;
-import static org.apache.storm.Config.SUPERVISOR_WORKER_TIMEOUT_SECS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyMapOf;
-import static org.mockito.Matchers.anySetOf;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import org.apache.storm.daemon.logviewer.testsupport.MockDirectoryBuilder;
+import org.apache.storm.daemon.logviewer.testsupport.MockFileBuilder;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Seq;
+import org.junit.Test;
+import org.mockito.internal.util.collections.Sets;
public class LogCleanerTest {
/**
@@ -84,10 +83,10 @@ public class LogCleanerTest {
LogCleaner logCleaner = new LogCleaner(conf, workerLogs, mockDirectoryCleaner, null);
- long nowMillis = Time.currentTimeMillis();
- long cutoffMillis = logCleaner.cleanupCutoffAgeMillis(nowMillis);
- long oldMtimeMillis = cutoffMillis - 500;
- long newMtimeMillis = cutoffMillis + 500;
+ final long nowMillis = Time.currentTimeMillis();
+ final long cutoffMillis = logCleaner.cleanupCutoffAgeMillis(nowMillis);
+ final long oldMtimeMillis = cutoffMillis - 500;
+ final long newMtimeMillis = cutoffMillis + 500;
List<File> matchingFiles = new ArrayList<>();
matchingFiles.add(new MockDirectoryBuilder().setDirName("3031").setMtime(oldMtimeMillis).build());
@@ -278,7 +277,7 @@ public class LogCleanerTest {
}
/**
- * cleanup function forceDeletes files of dead workers
+ * cleanup function forceDeletes files of dead workers.
*/
@Test
public void testCleanupFn() throws IOException {
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java
index 3d0e43e..63363fa 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java
@@ -18,14 +18,6 @@
package org.apache.storm.daemon.logviewer.utils;
-import org.apache.storm.daemon.logviewer.testsupport.ArgumentsVerifier;
-import org.apache.storm.utils.Utils;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
import static org.apache.storm.Config.NIMBUS_ADMINS;
import static org.apache.storm.Config.TOPOLOGY_GROUPS;
import static org.apache.storm.Config.TOPOLOGY_USERS;
@@ -38,6 +30,14 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.storm.daemon.logviewer.testsupport.ArgumentsVerifier;
+import org.apache.storm.utils.Utils;
+import org.junit.Test;
+
public class ResourceAuthorizerTest {
/**
@@ -172,11 +172,11 @@ public class ResourceAuthorizerTest {
private void verifyStubMethodsAreCalledProperly(ResourceAuthorizer authorizer) {
ArgumentsVerifier.verifyFirstCallArgsForSingleArgMethod(
- captor -> verify(authorizer, times(2)).getLogUserGroupWhitelist(captor.capture()),
- String.class, "non-blank-fname");
+ captor -> verify(authorizer, times(2)).getLogUserGroupWhitelist(captor.capture()),
+ String.class, "non-blank-fname");
ArgumentsVerifier.verifyFirstCallArgsForSingleArgMethod(
- captor -> verify(authorizer).getUserGroups(captor.capture()),
- String.class, "alice");
+ captor -> verify(authorizer).getUserGroups(captor.capture()),
+ String.class, "alice");
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6a041a98/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java
index b8b5d8c..30b1678 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java
@@ -18,11 +18,10 @@
package org.apache.storm.daemon.logviewer.utils;
-import org.apache.storm.daemon.logviewer.testsupport.MockDirectoryBuilder;
-import org.apache.storm.daemon.logviewer.testsupport.MockFileBuilder;
-import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.utils.Utils;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyMapOf;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
@@ -30,10 +29,11 @@ import java.util.Collections;
import java.util.Map;
import java.util.Optional;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyMapOf;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import org.apache.storm.daemon.logviewer.testsupport.MockDirectoryBuilder;
+import org.apache.storm.daemon.logviewer.testsupport.MockFileBuilder;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.utils.Utils;
+import org.junit.Test;
public class WorkerLogsTest {
[14/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* addressed received review comments so far
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f9ed5d43
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f9ed5d43
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f9ed5d43
Branch: refs/heads/master
Commit: f9ed5d432ce433de7d2f7b475bf77e02ef563341
Parents: 6a041a9
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Jul 20 12:16:59 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Jul 20 12:16:59 2017 +0900
----------------------------------------------------------------------
.../apache/storm/daemon/logviewer/LogviewerServer.java | 10 ----------
.../daemon/logviewer/handler/LogviewerLogPageHandler.java | 2 +-
2 files changed, 1 insertion(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f9ed5d43/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
index 4f9834d..0802015 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java
@@ -80,7 +80,6 @@ public class LogviewerServer implements AutoCloseable {
final Boolean httpsWantClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_WANT_CLIENT_AUTH));
final Boolean httpsNeedClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_NEED_CLIENT_AUTH));
- //TODO a better way to do this would be great.
LogviewerApplication.setup(conf);
ret = UIHelpers.jettyCreateServer(logviewerHttpPort, null, httpsPort);
@@ -150,15 +149,6 @@ public class LogviewerServer implements AutoCloseable {
}
/**
- * @return The port the HTTP server is listening on. Not available until {@link #start() } has run.
- */
- public int getHttpServerPort() {
- assert httpServer.getConnectors().length == 1;
-
- return httpServer.getConnectors()[0].getLocalPort();
- }
-
- /**
* Main method to start the server.
*/
public static void main(String [] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/storm/blob/f9ed5d43/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
index 04b1a13..47fa4b3 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
@@ -132,7 +132,7 @@ public class LogviewerLogPageHandler {
if (port == null) {
fileResults = new ArrayList<>();
- File topoDir = new File(logRoot + Utils.FILE_PATH_SEPARATOR + topologyId);
+ File topoDir = new File(logRoot, topologyId);
if (topoDir.exists()) {
File[] topoDirFiles = topoDir.listFiles();
if (topoDirFiles != null) {
[03/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
new file mode 100644
index 0000000..29dff1b
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
@@ -0,0 +1,854 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.handler;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.logviewer.LogviewerConstant;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Seq;
+import org.jooq.lambda.Unchecked;
+import org.jooq.lambda.tuple.Tuple3;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.attribute.FileAttribute;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static java.util.stream.Collectors.joining;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.internal.verification.VerificationModeFactory.times;
+
+@RunWith(Enclosed.class)
+public class LogviewerLogSearchHandlerTest {
+
+ public static class SearchViaRestApi {
+ private String pattern = "needle";
+ private String expectedHost = "dev.null.invalid";
+ private Integer expectedPort = 8888;
+ private String logviewerUrlPrefix = "http://" + expectedHost + ":" + expectedPort;
+
+ /*
+ When we click a link to the logviewer, we expect the match line to
+ be somewhere near the middle of the page. So we subtract half of
+ the default page length from the offset at which we found the
+ match.
+ */
+ private Function<Integer, Integer> expOffsetFn = arg -> (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2 - arg);
+
+ @Test(expected = RuntimeException.class)
+ public void testSearchViaRestApiThrowsIfBogusFileIsGiven() throws InvalidRequestException {
+ LogviewerLogSearchHandler handler = getSearchHandler();
+ handler.substringSearch(null, "a string");
+ }
+
+ @Test
+ public void testLogviewerLinkCentersTheMatchInThePage() throws UnknownHostException {
+ String expectedFname = "foobar.log";
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ String actualUrl = handler.urlToMatchCenteredInLogPage(new byte[42], expectedFname, 27526, 8888);
+
+ assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/log?file=" + expectedFname +
+ "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ @Test
+ public void testLogviewerLinkCentersTheMatchInThePageDaemon() throws UnknownHostException {
+ String expectedFname = "foobar.log";
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ String actualUrl = handler.urlToMatchCenteredInLogPage(new byte[42], expectedFname, 27526, 8888);
+
+ assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/daemonlog?file=" + expectedFname +
+ "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ @Test
+ public void testReturnsCorrectBeforeAndAfterContext() throws InvalidRequestException, UnknownHostException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "logviewer-search-context-tests.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(0, "",
+ " needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle ",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(7, "needle ",
+ "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle needle\n",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(127,
+ "needle needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+ " needle\n",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ // FIXME: currently fail on this...
+ matches.add(buildMatchData(134,
+ " needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle ",
+ "\n",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", matches);
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ @Test
+ public void testAreallySmallLogFile() throws InvalidRequestException, UnknownHostException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "small-worker.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(7, "000000 ",
+ " 000000\n",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", matches);
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ @Test
+ public void testAreallySmallLogDaemonFile() throws InvalidRequestException, UnknownHostException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "small-worker.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearchDaemonLog(file, pattern);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "yes");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(7, "000000 ",
+ " 000000\n",
+ pattern,
+ "/api/v1/daemonlog?file=" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", matches);
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ // FIXME: failing now
+ @Test
+ public void testNoOffsetReturnedWhenFileEndsOnBufferOffset() throws InvalidRequestException, UnknownHostException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "test-3072.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+ Map<String, Object> searchResult2 = handler.substringSearch(file, pattern, 1);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(3066,
+ Seq.range(0, 128).map(x -> ".").collect(joining()),
+ "",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", matches);
+
+ assertEquals(expected, searchResult);
+ assertEquals(expected, searchResult2);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ @Test
+ public void testNextByteOffsetsAreCorrectForEachMatch() throws UnknownHostException, InvalidRequestException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "test-worker.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+
+ List<Tuple3<Integer, Integer, Integer>> dataAndExpected = new ArrayList<>();
+ // numMatchesSought, numMatchesFound, expectedNextByteOffset
+ dataAndExpected.add(new Tuple3<>(1, 1, 11));
+ dataAndExpected.add(new Tuple3<>(2, 2, 2042));
+ dataAndExpected.add(new Tuple3<>(3, 3, 2052));
+ dataAndExpected.add(new Tuple3<>(4, 4, 3078));
+ dataAndExpected.add(new Tuple3<>(5, 5, 3196));
+ dataAndExpected.add(new Tuple3<>(6, 6, 3202));
+ dataAndExpected.add(new Tuple3<>(7, 7, 6252));
+ dataAndExpected.add(new Tuple3<>(8, 8, 6321));
+ dataAndExpected.add(new Tuple3<>(9, 9, 6397));
+ dataAndExpected.add(new Tuple3<>(10, 10, 6476));
+ dataAndExpected.add(new Tuple3<>(11, 11, 6554));
+ dataAndExpected.add(new Tuple3<>(12, 12, null));
+ dataAndExpected.add(new Tuple3<>(13, 12, null));
+
+ dataAndExpected.forEach(Unchecked.consumer(data -> {
+ Map<String, Object> result = handler.substringSearch(file, pattern, data.v1());
+ assertEquals(data.v3(), result.get("nextByteOffset"));
+ assertEquals(data.v2().intValue(), ((List) result.get("matches")).size());
+ }));
+
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 7);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+ expected.put("nextByteOffset", 6252);
+
+ // FIXME: currently failing on below...
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(5,
+ "Test ",
+ " is near the beginning of the file.\nThis file assumes a buffer size of 2048 bytes, a max search string size of 1024 bytes, and a",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(2036,
+ "ng 146\npadding 147\npadding 148\npadding 149\npadding 150\npadding 151\npadding 152\npadding 153\nNear the end of a 1024 byte block, a ",
+ ".\nA needle that straddles a 1024 byte boundary should also be detected.\n\npadding 157\npadding 158\npadding 159\npadding 160\npadding",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(2046,
+ "ding 147\npadding 148\npadding 149\npadding 150\npadding 151\npadding 152\npadding 153\nNear the end of a 1024 byte block, a needle.\nA ",
+ " that straddles a 1024 byte boundary should also be detected.\n\npadding 157\npadding 158\npadding 159\npadding 160\npadding 161\npaddi",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(3072,
+ "adding 226\npadding 227\npadding 228\npadding 229\npadding 230\npadding 231\npadding 232\npadding 233\npadding 234\npadding 235\n\n\nHere a ",
+ " occurs just after a 1024 byte boundary. It should have the correct context.\n\nText with two adjoining matches: needleneedle\n\npa",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(3190,
+ "\n\n\nHere a needle occurs just after a 1024 byte boundary. It should have the correct context.\n\nText with two adjoining matches: ",
+ "needle\n\npadding 243\npadding 244\npadding 245\npadding 246\npadding 247\npadding 248\npadding 249\npadding 250\npadding 251\npadding 252\n",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(3196,
+ "e a needle occurs just after a 1024 byte boundary. It should have the correct context.\n\nText with two adjoining matches: needle",
+ "\n\npadding 243\npadding 244\npadding 245\npadding 246\npadding 247\npadding 248\npadding 249\npadding 250\npadding 251\npadding 252\npaddin",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(6246,
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n\nHere are four non-ascii 1-byte UTF-8 characters: αβγδε\n\n",
+ "\n\nHere are four printable 2-byte UTF-8 characters: ¡¢£¤¥\n\nneedle\n\n\n\nHere are four printable 3-byte UTF-8 characters: ऄअ",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", matches);
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ // FIXME: this is failing...
+ @Test
+ public void testCorrectMatchOffsetIsReturnedWhenSkippingBytes() throws InvalidRequestException, UnknownHostException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "test-worker.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+
+ int startByteOffset = 3197;
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 1, startByteOffset);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", startByteOffset);
+ expected.put("nextByteOffset", 6252);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(6246,
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n\nHere are four non-ascii 1-byte UTF-8 characters: αβγδε\n\n",
+ "\n\nHere are four printable 2-byte UTF-8 characters: ¡¢£¤¥\n\nneedle\n\n\n\nHere are four printable 3-byte UTF-8 characters: ऄअ",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", matches);
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ // FIXME: this is failing...
+ // java.lang.StringIndexOutOfBoundsException: String index out of range: -39
+ @Test
+ public void testAnotherPatterns1() throws UnknownHostException, InvalidRequestException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "test-worker.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+
+ String pattern = Seq.range(0, 1024).map(x -> "X").collect(joining());
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 2);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+ expected.put("nextByteOffset", 6183);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(4075,
+ "\n\nThe following match of 1024 bytes completely fills half the byte buffer. It is a search substring of the maximum size......\n\n",
+ "\nThe following max-size match straddles a 1024 byte buffer.\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ matches.add(buildMatchData(5159,
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\nThe following max-size match straddles a 1024 byte buffer.\n",
+ "\n\nHere are four non-ascii 1-byte UTF-8 characters: αβγδε\n\nneedle\n\nHere are four printable 2-byte UTF-8 characters: ¡¢£¤",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", Collections.emptyList());
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ // FIXME: this is failing...
+ @Test
+ public void testAnotherPatterns2() throws UnknownHostException, InvalidRequestException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "test-worker.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+
+ String pattern = "𐄀𐄁𐄂";
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern, 1);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+ expected.put("nextByteOffset", 7176);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+
+ matches.add(buildMatchData(7164,
+ "padding 372\npadding 373\npadding 374\npadding 375\n\nThe following tests multibyte UTF-8 Characters straddling the byte boundary: ",
+ "\n\nneedle",
+ pattern,
+ "/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
+ ));
+
+ expected.put("matches", matches);
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ @Test
+ public void testReturnsZeroMatchesForUnseenPattern() throws UnknownHostException, InvalidRequestException {
+ Utils prevUtils = null;
+ try {
+ Utils mockedUtil = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockedUtil);
+
+ String pattern = "Not There";
+
+ when(mockedUtil.hostname()).thenReturn(expectedHost);
+
+ File file = new File(String.join(File.separator, "src", "test", "resources"),
+ "test-worker.log.test");
+
+ LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort);
+
+ Map<String, Object> searchResult = handler.substringSearch(file, pattern);
+
+ Map<String, Object> expected = new HashMap<>();
+ expected.put("isDaemon", "no");
+ expected.put("searchString", pattern);
+ expected.put("startByteOffset", 0);
+
+ expected.put("matches", Collections.emptyList());
+
+ assertEquals(expected, searchResult);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ private Map<String, Object> buildMatchData(int byteOffset, String beforeString, String afterString,
+ String matchString, String logviewerUrlPath) {
+ Map<String, Object> match = new HashMap<>();
+ match.put("byteOffset", byteOffset);
+ match.put("beforeString", beforeString);
+ match.put("afterString", afterString);
+ match.put("matchString", matchString);
+ match.put("logviewerURL", logviewerUrlPrefix + logviewerUrlPath);
+ return match;
+ }
+ }
+
+ public static class FindNMatchesTest {
+ /**
+ * find-n-matches looks through logs properly.
+ */
+ @Test
+ public void testFindNMatches() {
+ List<File> files = new ArrayList<>();
+ files.add(new File(String.join(File.separator, "src", "test", "resources"),
+ "logviewer-search-context-tests.log.test"));
+ files.add(new File(String.join(File.separator, "src", "test", "resources"),
+ "logviewer-search-context-tests.log.gz"));
+
+ LogviewerLogSearchHandler handler = getSearchHandler();
+
+ List<Map<String, Object>> matches1 = handler.findNMatches(files, 20, 0, 0, "needle").getMatches();
+ List<Map<String, Object>> matches2 = handler.findNMatches(files, 20, 0, 126, "needle").getMatches();
+ List<Map<String, Object>> matches3 = handler.findNMatches(files, 20, 1, 0, "needle").getMatches();
+
+ assertEquals(2, matches1.size());
+ assertEquals(4, ((List) matches1.get(0).get("matches")).size());
+ assertEquals(4, ((List) matches1.get(1).get("matches")).size());
+ assertEquals("test/resources/logviewer-search-context-tests.log.test", matches1.get(0).get("fileName"));
+ assertEquals("test/resources/logviewer-search-context-tests.log.gz", matches1.get(1).get("fileName"));
+
+ assertEquals(2, ((List) matches2.get(0).get("matches")).size());
+ assertEquals(4, ((List) matches2.get(1).get("matches")).size());
+
+ assertEquals(1, matches3.size());
+ assertEquals(4, ((List) matches3.get(0).get("matches")).size());
+ }
+ }
+
+ public static class TestDeepSearchLogs {
+
+ private List<File> logFiles;
+ private String topoPath;
+
+ @Before
+ public void setUp() throws IOException {
+ logFiles = new ArrayList<>();
+ logFiles.add(new File(String.join(File.separator, "src", "test", "resources"),
+ "logviewer-search-context-tests.log.test"));
+ logFiles.add(new File(String.join(File.separator, "src", "test", "resources"),
+ "logviewer-search-context-tests.log.gz"));
+
+ FileAttribute[] attrs = new FileAttribute[0];
+ topoPath = Files.createTempDirectory("topoA", attrs).toFile().getCanonicalPath();
+ new File(topoPath, "6400").createNewFile();
+ new File(topoPath, "6500").createNewFile();
+ new File(topoPath, "6600").createNewFile();
+ new File(topoPath, "6700").createNewFile();
+ }
+
+ @After
+ public void tearDown() {
+ if (StringUtils.isNotEmpty(topoPath)) {
+ try {
+ Utils.forceDelete(topoPath);
+ } catch (IOException e) {
+ // ignore...
+ }
+ }
+ }
+
+ @Test
+ public void testAllPortsAndSearchArchivedIsTrue() throws IOException {
+ LogviewerLogSearchHandler handler = getStubbedSearchHandler();
+
+ handler.deepSearchLogsForTopology("", null, "search", "20", "*", "20", "199", true, null, null);
+
+ ArgumentCaptor<List> files = ArgumentCaptor.forClass(List.class);
+ ArgumentCaptor<Integer> numMatches = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> fileOffset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> offset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<String> search = ArgumentCaptor.forClass(String.class);
+
+ verify(handler, times(4)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(),
+ offset.capture(), search.capture());
+ verify(handler, times(4)).logsForPort(anyString(), any(File.class));
+
+ // File offset and byte offset should always be zero when searching multiple workers (multiple ports).
+ assertEquals(logFiles, files.getAllValues().get(0));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(0));
+ assertEquals("search", search.getAllValues().get(0));
+ assertEquals(logFiles, files.getAllValues().get(0));
+
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(1));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(1));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(1));
+ assertEquals("search", search.getAllValues().get(1));
+ assertEquals(logFiles, files.getAllValues().get(1));
+
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(2));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(2));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(2));
+ assertEquals("search", search.getAllValues().get(2));
+ assertEquals(logFiles, files.getAllValues().get(2));
+
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(3));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(3));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(3));
+ assertEquals("search", search.getAllValues().get(3));
+ }
+
+ @Test
+ public void testAllPortsAndSearchArchivedIsFalse() throws IOException {
+ LogviewerLogSearchHandler handler = getStubbedSearchHandler();
+
+ handler.deepSearchLogsForTopology("", null, "search", "20", null, "20", "199", false, null, null);
+
+ ArgumentCaptor<List> files = ArgumentCaptor.forClass(List.class);
+ ArgumentCaptor<Integer> numMatches = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> fileOffset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> offset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<String> search = ArgumentCaptor.forClass(String.class);
+
+ verify(handler, times(4)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(),
+ offset.capture(), search.capture());
+ verify(handler, times(4)).logsForPort(anyString(), any(File.class));
+
+ // File offset and byte offset should always be zero when searching multiple workers (multiple ports).
+ assertEquals(Collections.singletonList(logFiles.get(0)), files.getAllValues().get(0));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(0));
+ assertEquals("search", search.getAllValues().get(0));
+
+ assertEquals(Collections.singletonList(logFiles.get(0)), files.getAllValues().get(1));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(1));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(1));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(1));
+ assertEquals("search", search.getAllValues().get(1));
+
+ assertEquals(Collections.singletonList(logFiles.get(0)), files.getAllValues().get(2));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(2));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(2));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(2));
+ assertEquals("search", search.getAllValues().get(2));
+
+ assertEquals(Collections.singletonList(logFiles.get(0)), files.getAllValues().get(3));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(3));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(3));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(3));
+ assertEquals("search", search.getAllValues().get(3));
+ }
+
+ @Test
+ public void testOnePortAndSearchArchivedIsTrueAndNotFileOffset() throws IOException {
+ LogviewerLogSearchHandler handler = getStubbedSearchHandler();
+
+ handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "0", "0", true, null, null);
+
+ ArgumentCaptor<List> files = ArgumentCaptor.forClass(List.class);
+ ArgumentCaptor<Integer> numMatches = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> fileOffset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> offset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<String> search = ArgumentCaptor.forClass(String.class);
+
+ verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(),
+ offset.capture(), search.capture());
+ verify(handler, times(2)).logsForPort(anyString(), any(File.class));
+
+ assertEquals(logFiles, files.getAllValues().get(0));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(0));
+ assertEquals("search", search.getAllValues().get(0));
+ }
+
+ @Test
+ public void testOnePortAndSearchArchivedIsTrueAndFileOffsetIs1() throws IOException {
+ LogviewerLogSearchHandler handler = getStubbedSearchHandler();
+
+ handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "0", true, null, null);
+
+ ArgumentCaptor<List> files = ArgumentCaptor.forClass(List.class);
+ ArgumentCaptor<Integer> numMatches = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> fileOffset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> offset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<String> search = ArgumentCaptor.forClass(String.class);
+
+ verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(),
+ offset.capture(), search.capture());
+ verify(handler, times(2)).logsForPort(anyString(), any(File.class));
+
+ assertEquals(logFiles, files.getAllValues().get(0));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(0));
+ assertEquals(Integer.valueOf(1), fileOffset.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(0));
+ assertEquals("search", search.getAllValues().get(0));
+ }
+
+ @Test
+ public void testOnePortAndSearchArchivedIsFalseAndFileOffsetIs1() throws IOException {
+ LogviewerLogSearchHandler handler = getStubbedSearchHandler();
+
+ handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "0", false, null, null);
+
+ ArgumentCaptor<List> files = ArgumentCaptor.forClass(List.class);
+ ArgumentCaptor<Integer> numMatches = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> fileOffset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> offset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<String> search = ArgumentCaptor.forClass(String.class);
+
+ verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(),
+ offset.capture(), search.capture());
+ verify(handler, times(2)).logsForPort(anyString(), any(File.class));
+
+ // File offset should be zero, since search-archived is false.
+ assertEquals(Collections.singletonList(logFiles.get(0)), files.getAllValues().get(0));
+ assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(0));
+ assertEquals(Integer.valueOf(0), offset.getAllValues().get(0));
+ assertEquals("search", search.getAllValues().get(0));
+ }
+
+ @Test
+ public void testOnePortAndSearchArchivedIsTrueAndFileOffsetIs1AndByteOffsetIs100() throws IOException {
+ LogviewerLogSearchHandler handler = getStubbedSearchHandler();
+
+ handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "100", true, null, null);
+
+ verify(handler, times(1)).findNMatches(anyListOf(File.class), anyInt(), anyInt(), anyInt(), anyString());
+ verify(handler, times(2)).logsForPort(anyString(), any(File.class));
+ }
+
+ @Test
+ public void testBadPortAndSearchArchivedIsFalseAndFileOffsetIs1() throws IOException {
+ LogviewerLogSearchHandler handler = getStubbedSearchHandler();
+
+ handler.deepSearchLogsForTopology("", null, "search", "20", "2700", "1", "0", false, null, null);
+
+ ArgumentCaptor<List> files = ArgumentCaptor.forClass(List.class);
+ ArgumentCaptor<Integer> numMatches = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> fileOffset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<Integer> offset = ArgumentCaptor.forClass(Integer.class);
+ ArgumentCaptor<String> search = ArgumentCaptor.forClass(String.class);
+
+ // Called with a bad port (not in the config) No searching should be done.
+ verify(handler, never()).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(),
+ offset.capture(), search.capture());
+ verify(handler, never()).logsForPort(anyString(), any(File.class));
+ }
+
+ private LogviewerLogSearchHandler getStubbedSearchHandler() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+ LogviewerLogSearchHandler handler = new LogviewerLogSearchHandler(stormConf, topoPath, null,
+ new ResourceAuthorizer(stormConf));
+ handler = spy(handler);
+
+ doReturn(logFiles).when(handler).logsForPort(anyString(), any(File.class));
+ doAnswer(invocationOnMock -> {
+ Object[] arguments = invocationOnMock.getArguments();
+ int fileOffset = (Integer) arguments[2];
+ String search = (String) arguments[4];
+
+ return new LogviewerLogSearchHandler.Matched(fileOffset, search, Collections.emptyList());
+ }).when(handler).findNMatches(anyListOf(File.class), anyInt(), anyInt(), anyInt(), anyString());
+
+ return handler;
+ }
+ }
+
+ private static LogviewerLogSearchHandler getSearchHandler() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+ return new LogviewerLogSearchHandler(stormConf, null, null,
+ new ResourceAuthorizer(stormConf));
+ }
+
+ private static LogviewerLogSearchHandler getSearchHandlerWithPort(int port) {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+ stormConf.put(DaemonConfig.LOGVIEWER_PORT, port);
+ return new LogviewerLogSearchHandler(stormConf, null, null,
+ new ResourceAuthorizer(stormConf));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java
new file mode 100644
index 0000000..e52c9db
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.testsupport;
+
+import org.mockito.ArgumentCaptor;
+
+import java.util.function.Consumer;
+
+import static org.junit.Assert.assertEquals;
+
+public class ArgumentsVerifier {
+ public static <T> void verifyFirstCallArgsForSingleArgMethod(Consumer<ArgumentCaptor<T>> verifyConsumer,
+ Class<T> argClazz, T expectedArg) {
+ ArgumentCaptor<T> captor = ArgumentCaptor.forClass(argClazz);
+ verifyConsumer.accept(captor);
+ assertEquals(expectedArg, captor.getAllValues().get(0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java
new file mode 100644
index 0000000..7215456
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockDirectoryBuilder.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.testsupport;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MockDirectoryBuilder {
+ private String dirName;
+ private long mtime;
+ private File[] files;
+
+ public MockDirectoryBuilder() {
+ this.dirName = "adir";
+ this.mtime = 1;
+ this.files = new File[]{};
+ }
+
+ public MockDirectoryBuilder setDirName(String name) {
+ this.dirName = name;
+ return this;
+ }
+
+ public MockDirectoryBuilder setMtime(long mtime) {
+ this.mtime = mtime;
+ return this;
+ }
+
+ public MockDirectoryBuilder setFiles(File[] files) {
+ this.files = files;
+ return this;
+ }
+
+ public File build() {
+ File mockFile = mock(File.class);
+ when(mockFile.getName()).thenReturn(dirName);
+ when(mockFile.lastModified()).thenReturn(mtime);
+ when(mockFile.isFile()).thenReturn(false);
+ when(mockFile.listFiles()).thenReturn(files);
+ try {
+ when(mockFile.getCanonicalPath()).thenReturn(dirName);
+ } catch (IOException e) {
+ // we're making mock, ignoring...
+ }
+ return mockFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java
new file mode 100644
index 0000000..385eec4
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockFileBuilder.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.testsupport;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MockFileBuilder {
+ private String fileName;
+ private long mtime;
+ private long length;
+
+ public MockFileBuilder() {
+ this.fileName = "afile";
+ this.mtime = 1;
+ this.length = 10 * 1024 * 1024 * 1024;
+ }
+
+ public MockFileBuilder setFileName(String fileName) {
+ this.fileName = fileName;
+ return this;
+ }
+
+ public MockFileBuilder setMtime(long mtime) {
+ this.mtime = mtime;
+ return this;
+ }
+
+ public MockFileBuilder setLength(long length) {
+ this.length = length;
+ return this;
+ }
+
+ public File build() {
+ File mockFile = mock(File.class);
+ when(mockFile.getName()).thenReturn(fileName);
+ when(mockFile.lastModified()).thenReturn(mtime);
+ when(mockFile.isFile()).thenReturn(true);
+ try {
+ when(mockFile.getCanonicalPath()).thenReturn("/mock/canonical/path/to/" + fileName);
+ } catch (IOException e) {
+ // we're making mock, ignoring...
+ }
+ when(mockFile.length()).thenReturn(length);
+ return mockFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
new file mode 100644
index 0000000..0c7d69c
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.utils;
+
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.daemon.logviewer.testsupport.MockDirectoryBuilder;
+import org.apache.storm.daemon.logviewer.testsupport.MockFileBuilder;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Seq;
+import org.junit.Test;
+import org.mockito.internal.util.collections.Sets;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.storm.Config.SUPERVISOR_WORKER_TIMEOUT_SECS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyMapOf;
+import static org.mockito.Matchers.anySetOf;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class LogCleanerTest {
+ /**
+ * Log file filter selects the correct worker-log dirs for purge.
+ */
+ @Test
+ public void testMkFileFilterForLogCleanup() throws IOException {
+ DirectoryCleaner mockDirectoryCleaner = mock(DirectoryCleaner.class);
+ when(mockDirectoryCleaner.getStreamForDirectory(any(File.class))).thenAnswer(invocationOnMock -> {
+ File file = (File) invocationOnMock.getArguments()[0];
+ List<Path> paths = Arrays.stream(file.listFiles()).map(f -> mkMockPath(f)).collect(toList());
+ return mkDirectoryStream(paths);
+ });
+
+ // this is to read default value for other configurations
+ Map<String, Object> conf = Utils.readStormConfig();
+ conf.put(LOGVIEWER_CLEANUP_AGE_MINS, 60);
+ conf.put(LOGVIEWER_CLEANUP_INTERVAL_SECS, 300);
+
+ LogCleaner logCleaner = new LogCleaner(conf, mockDirectoryCleaner);
+
+ long nowMillis = Time.currentTimeMillis();
+ long cutoffMillis = new LogCleaner(conf, mockDirectoryCleaner).cleanupCutoffAgeMillis(nowMillis);
+ long oldMtimeMillis = cutoffMillis - 500;
+ long newMtimeMillis = cutoffMillis + 500;
+
+ List<File> matchingFiles = new ArrayList<>();
+ matchingFiles.add(new MockDirectoryBuilder().setDirName("3031").setMtime(oldMtimeMillis).build());
+ matchingFiles.add(new MockDirectoryBuilder().setDirName("3032").setMtime(oldMtimeMillis).build());
+ matchingFiles.add(new MockDirectoryBuilder().setDirName("7077").setMtime(oldMtimeMillis).build());
+
+ List<File> excludedFiles = new ArrayList<>();
+ excludedFiles.add(new MockFileBuilder().setFileName("oldlog-1-2-worker-.log").setMtime(oldMtimeMillis).build());
+ excludedFiles.add(new MockFileBuilder().setFileName("newlog-1-2-worker-.log").setMtime(newMtimeMillis).build());
+ excludedFiles.add(new MockFileBuilder().setFileName("some-old-file.txt").setMtime(oldMtimeMillis).build());
+ excludedFiles.add(new MockFileBuilder().setFileName("olddir-1-2-worker.log").setMtime(newMtimeMillis).build());
+ excludedFiles.add(new MockDirectoryBuilder().setDirName("metadata").setMtime(newMtimeMillis).build());
+ excludedFiles.add(new MockDirectoryBuilder().setDirName("newdir").setMtime(newMtimeMillis).build());
+
+ FileFilter fileFilter = logCleaner.mkFileFilterForLogCleanup(nowMillis);
+
+ assertTrue(matchingFiles.stream().allMatch(fileFilter::accept));
+ assertTrue(excludedFiles.stream().noneMatch(fileFilter::accept));
+ }
+
+ /**
+ * cleaner deletes oldest files in each worker dir if files are larger than per-dir quota.
+ */
+ @Test
+ public void testPerWorkerDirectoryCleanup() throws IOException {
+ Utils prevUtils = null;
+ try {
+ Utils mockUtils = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockUtils);
+
+ DirectoryCleaner mockDirectoryCleaner = mock(DirectoryCleaner.class);
+ when(mockDirectoryCleaner.getStreamForDirectory(any(File.class))).thenAnswer(invocationOnMock -> {
+ File file = (File) invocationOnMock.getArguments()[0];
+ List<Path> paths = Arrays.stream(file.listFiles()).map(f -> mkMockPath(f)).collect(toList());
+ return mkDirectoryStream(paths);
+ });
+ when(mockDirectoryCleaner.deleteOldestWhileTooLarge(anyListOf(File.class), anyLong(), anyBoolean(), anySetOf(String.class)))
+ .thenCallRealMethod();
+
+ Map<String, Object> conf = Utils.readStormConfig();
+ LogCleaner logCleaner = new LogCleaner(conf, mockDirectoryCleaner);
+
+ long nowMillis = Time.currentTimeMillis();
+
+ List<File> files1 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("A" + idx)
+ .setMtime(nowMillis + (100 * idx)).setLength(200).build())
+ .collect(toList());
+ List<File> files2 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("B" + idx)
+ .setMtime(nowMillis + (100 * idx)).setLength(200).build())
+ .collect(toList());
+ List<File> files3 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("C" + idx)
+ .setMtime(nowMillis + (100 * idx)).setLength(200).build())
+ .collect(toList());
+ File port1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port1")
+ .setFiles(files1.toArray(new File[]{})).build();
+ File port2Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port2")
+ .setFiles(files2.toArray(new File[]{})).build();
+ File port3Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo2/port3")
+ .setFiles(files3.toArray(new File[]{})).build();
+
+ File[] topo1Files = new File[] { port1Dir, port2Dir };
+ File[] topo2Files = new File[] { port3Dir };
+ File topo1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1")
+ .setFiles(topo1Files).build();
+ File topo2Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo2")
+ .setFiles(topo2Files).build();
+
+ File[] rootFiles = new File[] { topo1Dir, topo2Dir };
+ File rootDir = new MockDirectoryBuilder().setDirName("/workers-artifacts")
+ .setFiles(rootFiles).build();
+
+ List<Integer> deletedFiles = logCleaner.perWorkerDirCleanup(rootDir, 1200, mockDirectoryCleaner);
+ assertEquals(Integer.valueOf(4), deletedFiles.get(0));
+ assertEquals(Integer.valueOf(4), deletedFiles.get(1));
+ assertEquals(Integer.valueOf(4), deletedFiles.get(deletedFiles.size() - 1));
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ @Test
+ public void testGlobalLogCleanup() throws Exception {
+ Utils prevUtils = null;
+ try {
+ Utils mockUtils = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockUtils);
+
+ DirectoryCleaner mockDirectoryCleaner = mock(DirectoryCleaner.class);
+ when(mockDirectoryCleaner.getStreamForDirectory(any(File.class))).thenAnswer(invocationOnMock -> {
+ File file = (File) invocationOnMock.getArguments()[0];
+ List<Path> paths = Arrays.stream(file.listFiles()).map(f -> mkMockPath(f)).collect(toList());
+ return mkDirectoryStream(paths);
+ });
+ when(mockDirectoryCleaner.deleteOldestWhileTooLarge(anyListOf(File.class), anyLong(), anyBoolean(), anySetOf(String.class)))
+ .thenCallRealMethod();
+
+ Map<String, Object> conf = Utils.readStormConfig();
+
+ LogCleaner logCleaner = new LogCleaner(conf, mockDirectoryCleaner) {
+ @Override
+ SortedSet<String> getAliveWorkerDirs(File rootDir) throws Exception {
+ return new TreeSet<>(Collections.singletonList("/workers-artifacts/topo1/port1"));
+ }
+ };
+
+ long nowMillis = Time.currentTimeMillis();
+
+ List<File> files1 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("A" + idx + ".log")
+ .setMtime(nowMillis + (100 * idx)).setLength(200).build())
+ .collect(toList());
+ List<File> files2 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("B" + idx)
+ .setMtime(nowMillis + (100 * idx)).setLength(200).build())
+ .collect(toList());
+ List<File> files3 = Seq.range(0, 10).map(idx -> new MockFileBuilder().setFileName("C" + idx)
+ .setMtime(nowMillis + (100 * idx)).setLength(200).build())
+ .collect(toList());
+
+ // note that port1Dir is active worker containing active logs
+ File port1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port1")
+ .setFiles(files1.toArray(new File[]{})).build();
+ File port2Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port2")
+ .setFiles(files2.toArray(new File[]{})).build();
+ File port3Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo2/port3")
+ .setFiles(files3.toArray(new File[]{})).build();
+
+ File[] topo1Files = new File[] { port1Dir, port2Dir };
+ File[] topo2Files = new File[] { port3Dir };
+ File topo1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1")
+ .setFiles(topo1Files).build();
+ File topo2Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo2")
+ .setFiles(topo2Files).build();
+
+ File[] rootFiles = new File[] { topo1Dir, topo2Dir };
+ File rootDir = new MockDirectoryBuilder().setDirName("/workers-artifacts")
+ .setFiles(rootFiles).build();
+
+ int deletedFiles = logCleaner.globalLogCleanup(rootDir, 2400, mockDirectoryCleaner);
+ assertEquals(18, deletedFiles);
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ /**
+ * Build up workerid-workerlogdir map for the old workers' dirs.
+ */
+ @Test
+ public void testIdentifyWorkerLogDirs() throws Exception {
+ File port1Dir = new MockDirectoryBuilder().setDirName("/workers-artifacts/topo1/port1").build();
+ File mockMetaFile = new MockFileBuilder().setFileName("worker.yaml").build();
+
+ String expId = "id12345";
+ Map<String, File> expected = Collections.singletonMap(expId, port1Dir);
+
+ try {
+ SupervisorUtils mockedSupervisorUtils = mock(SupervisorUtils.class);
+ SupervisorUtils.setInstance(mockedSupervisorUtils);
+
+ Map<String, Object> stormConf = Utils.readStormConfig();
+ LogCleaner logCleaner = new LogCleaner(stormConf, new DirectoryCleaner()) {
+ @Override
+ Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
+ return Optional.of(mockMetaFile);
+ }
+
+ @Override
+ String getWorkerIdFromMetadataFile(String metaFile) {
+ return expId;
+ }
+ };
+
+ when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMapOf(String.class, Object.class))).thenReturn(null);
+ assertEquals(expected, logCleaner.identifyWorkerLogDirs(Collections.singleton(port1Dir)));
+ } finally {
+ SupervisorUtils.resetInstance();
+ }
+ }
+
+ /**
+ * return directories for workers that are not alive.
+ */
+ @Test
+ public void testGetDeadWorkerDirs() throws Exception {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+ stormConf.put(SUPERVISOR_WORKER_TIMEOUT_SECS, 5);
+
+ LSWorkerHeartbeat hb = new LSWorkerHeartbeat();
+ hb.set_time_secs(1);
+
+ Map<String, LSWorkerHeartbeat> idToHb = Collections.singletonMap("42", hb);
+ int nowSecs = 2;
+ File unexpectedDir1 = new MockDirectoryBuilder().setDirName("dir1").build();
+ File expectedDir2 = new MockDirectoryBuilder().setDirName("dir2").build();
+ File expectedDir3 = new MockDirectoryBuilder().setDirName("dir3").build();
+ Set<File> logDirs = Sets.newSet(unexpectedDir1, expectedDir2, expectedDir3);
+
+ try {
+ SupervisorUtils mockedSupervisorUtils = mock(SupervisorUtils.class);
+ SupervisorUtils.setInstance(mockedSupervisorUtils);
+
+ LogCleaner logCleaner = new LogCleaner(stormConf, new DirectoryCleaner()) {
+ @Override
+ Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
+ Map<String, File> ret = new HashMap<>();
+ ret.put("42", unexpectedDir1);
+ ret.put("007", expectedDir2);
+ // this tests a directory with no yaml file thus no worker id
+ ret.put("", expectedDir3);
+
+ return ret;
+ }
+ };
+
+ when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMapOf(String.class, Object.class))).thenReturn(idToHb);
+ assertEquals(Sets.newSet(expectedDir2, expectedDir3), logCleaner.getDeadWorkerDirs(nowSecs, logDirs));
+ } finally {
+ SupervisorUtils.resetInstance();
+ }
+ }
+
+ /**
+ * cleanup function forceDeletes files of dead workers
+ */
+ @Test
+ public void testCleanupFn() throws IOException {
+ File mockFile1 = new MockFileBuilder().setFileName("delete-me1").build();
+ File mockFile2 = new MockFileBuilder().setFileName("delete-me2").build();
+
+ Utils prevUtils = null;
+ try {
+ Utils mockUtils = mock(Utils.class);
+ prevUtils = Utils.setInstance(mockUtils);
+
+ List<String> forceDeleteArgs = new ArrayList<>();
+ doAnswer(invocationOnMock -> {
+ String path = (String) invocationOnMock.getArguments()[0];
+ forceDeleteArgs.add(path);
+ return null;
+ }).when(mockUtils).forceDelete(anyString());
+
+ LogCleaner logCleaner = new LogCleaner(Utils.readStormConfig(), new DirectoryCleaner()) {
+ @Override
+ Set<File> selectDirsForCleanup(long nowMillis, String rootDir) {
+ return Collections.emptySet();
+ }
+
+ @Override
+ SortedSet<File> getDeadWorkerDirs(int nowSecs, Set<File> logDirs) throws Exception {
+ SortedSet<File> dirs = new TreeSet<>();
+ dirs.add(mockFile1);
+ dirs.add(mockFile2);
+ return dirs;
+ }
+
+ @Override
+ void cleanupEmptyTopoDirectory(File dir) throws IOException {
+ }
+ };
+
+ logCleaner.run();
+
+ assertEquals(2, forceDeleteArgs.size());
+ assertEquals(mockFile1.getCanonicalPath(), forceDeleteArgs.get(0));
+ assertEquals(mockFile2.getCanonicalPath(), forceDeleteArgs.get(1));
+ } finally {
+ Utils.setInstance(prevUtils);
+ }
+ }
+
+ private Path mkMockPath(File file) {
+ Path mockPath = mock(Path.class);
+ when(mockPath.toFile()).thenReturn(file);
+ return mockPath;
+ }
+
+ private DirectoryStream<Path> mkDirectoryStream(List<Path> listOfPaths) {
+ return new DirectoryStream<Path>() {
+ @Override
+ public Iterator<Path> iterator() {
+ return listOfPaths.iterator();
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java
new file mode 100644
index 0000000..3d0e43e
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.utils;
+
+import org.apache.storm.daemon.logviewer.testsupport.ArgumentsVerifier;
+import org.apache.storm.utils.Utils;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.storm.Config.NIMBUS_ADMINS;
+import static org.apache.storm.Config.TOPOLOGY_GROUPS;
+import static org.apache.storm.Config.TOPOLOGY_USERS;
+import static org.apache.storm.DaemonConfig.LOGS_USERS;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class ResourceAuthorizerTest {
+
+ /**
+ * allow cluster admin.
+ */
+ @Test
+ public void testAuthorizedLogUserAllowClusterAdmin() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+
+ Map<String, Object> conf = new HashMap<>(stormConf);
+ conf.put(NIMBUS_ADMINS, Collections.singletonList("alice"));
+
+ ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf));
+
+ doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet()))
+ .when(authorizer).getLogUserGroupWhitelist(anyString());
+
+ doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString());
+
+ assertTrue(authorizer.isAuthorizedLogUser("alice", "non-blank-fname"));
+
+ verifyStubMethodsAreCalledProperly(authorizer);
+ }
+
+ /**
+ * ignore any cluster-set topology.users topology.groups.
+ */
+ @Test
+ public void testAuthorizedLogUserIgnoreAnyClusterSetTopologyUsersAndTopologyGroups() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+
+ Map<String, Object> conf = new HashMap<>(stormConf);
+ conf.put(TOPOLOGY_USERS, Collections.singletonList("alice"));
+ conf.put(TOPOLOGY_GROUPS, Collections.singletonList("alice-group"));
+
+ ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf));
+
+ doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet()))
+ .when(authorizer).getLogUserGroupWhitelist(anyString());
+
+ doReturn(Collections.singleton("alice-group")).when(authorizer).getUserGroups(anyString());
+
+ assertFalse(authorizer.isAuthorizedLogUser("alice", "non-blank-fname"));
+
+ verifyStubMethodsAreCalledProperly(authorizer);
+ }
+
+ /**
+ * allow cluster logs user.
+ */
+ @Test
+ public void testAuthorizedLogUserAllowClusterLogsUser() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+
+ Map<String, Object> conf = new HashMap<>(stormConf);
+ conf.put(LOGS_USERS, Collections.singletonList("alice"));
+
+ ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf));
+
+ doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet()))
+ .when(authorizer).getLogUserGroupWhitelist(anyString());
+
+ doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString());
+
+ assertTrue(authorizer.isAuthorizedLogUser("alice", "non-blank-fname"));
+
+ verifyStubMethodsAreCalledProperly(authorizer);
+ }
+
+ /**
+ * allow whitelisted topology user.
+ */
+ @Test
+ public void testAuthorizedLogUserAllowWhitelistedTopologyUser() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+
+ Map<String, Object> conf = new HashMap<>(stormConf);
+
+ ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf));
+
+ doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.singleton("alice"), Collections.emptySet()))
+ .when(authorizer).getLogUserGroupWhitelist(anyString());
+
+ doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString());
+
+ assertTrue(authorizer.isAuthorizedLogUser("alice", "non-blank-fname"));
+
+ verifyStubMethodsAreCalledProperly(authorizer);
+ }
+
+ /**
+ * allow whitelisted topology group.
+ */
+ @Test
+ public void testAuthorizedLogUserAllowWhitelistedTopologyGroup() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+
+ Map<String, Object> conf = new HashMap<>(stormConf);
+
+ ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf));
+
+ doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.singleton("alice-group")))
+ .when(authorizer).getLogUserGroupWhitelist(anyString());
+
+ doReturn(Collections.singleton("alice-group")).when(authorizer).getUserGroups(anyString());
+
+ assertTrue(authorizer.isAuthorizedLogUser("alice", "non-blank-fname"));
+
+ verifyStubMethodsAreCalledProperly(authorizer);
+ }
+
+ /**
+ * disallow user not in nimbus admin, topo user, logs user, or whitelist.
+ */
+ @Test
+ public void testAuthorizedLogUserDisallowUserNotInNimbusAdminNorTopoUserNorLogsUserNotWhitelist() {
+ Map<String, Object> stormConf = Utils.readStormConfig();
+
+ Map<String, Object> conf = new HashMap<>(stormConf);
+
+ ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf));
+
+ doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet()))
+ .when(authorizer).getLogUserGroupWhitelist(anyString());
+
+ doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString());
+
+ assertFalse(authorizer.isAuthorizedLogUser("alice", "non-blank-fname"));
+
+ verifyStubMethodsAreCalledProperly(authorizer);
+ }
+
+ private void verifyStubMethodsAreCalledProperly(ResourceAuthorizer authorizer) {
+ ArgumentsVerifier.verifyFirstCallArgsForSingleArgMethod(
+ captor -> verify(authorizer, times(2)).getLogUserGroupWhitelist(captor.capture()),
+ String.class, "non-blank-fname");
+
+ ArgumentsVerifier.verifyFirstCallArgsForSingleArgMethod(
+ captor -> verify(authorizer).getUserGroups(captor.capture()),
+ String.class, "alice");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/resources/logviewer-search-context-tests.log.gz
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/resources/logviewer-search-context-tests.log.gz b/storm-webapp/src/test/resources/logviewer-search-context-tests.log.gz
new file mode 100644
index 0000000..5cf2a06
Binary files /dev/null and b/storm-webapp/src/test/resources/logviewer-search-context-tests.log.gz differ
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/resources/logviewer-search-context-tests.log.test
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/resources/logviewer-search-context-tests.log.test b/storm-webapp/src/test/resources/logviewer-search-context-tests.log.test
new file mode 100644
index 0000000..6e4d4af
--- /dev/null
+++ b/storm-webapp/src/test/resources/logviewer-search-context-tests.log.test
@@ -0,0 +1 @@
+needle needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle needle
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/resources/small-worker.log.test
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/resources/small-worker.log.test b/storm-webapp/src/test/resources/small-worker.log.test
new file mode 100644
index 0000000..27d61d1
--- /dev/null
+++ b/storm-webapp/src/test/resources/small-worker.log.test
@@ -0,0 +1 @@
+000000 needle 000000
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/resources/test-3072.log.test
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/resources/test-3072.log.test b/storm-webapp/src/test/resources/test-3072.log.test
new file mode 100644
index 0000000..56dc6f1
--- /dev/null
+++ b/storm-webapp/src/test/resources/test-3072.log.test
@@ -0,0 +1,3 @@
+This is a test log file of size 3072.
+
+.....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
.....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
.....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
....................................needle
\ No newline at end of file
[08/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* ported logviewer-test.clj
* TODO: some tests are failing, looking into it
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/44b268ba
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/44b268ba
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/44b268ba
Branch: refs/heads/master
Commit: 44b268badb7ee1a92b1cc8abf81dbb640fa9af19
Parents: 11a7905
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Jul 13 00:27:38 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Jul 14 12:11:41 2017 +0900
----------------------------------------------------------------------
bin/storm.py | 2 +-
.../clj/org/apache/storm/daemon/logviewer.clj | 1254 ------------------
.../dev/logviewer-search-context-tests.log.gz | Bin 72 -> 0 bytes
.../dev/logviewer-search-context-tests.log.test | 1 -
storm-core/src/dev/small-worker.log.test | 1 -
storm-core/src/dev/test-3072.log.test | 3 -
storm-core/src/dev/test-worker.log.test | 380 ------
.../apache/storm/daemon/DirectoryCleaner.java | 188 ---
.../clj/org/apache/storm/logviewer_test.clj | 824 ------------
.../daemon/logviewer/LogviewerConstant.java | 23 +
.../storm/daemon/logviewer/LogviewerServer.java | 172 +++
.../handler/LogviewerLogDownloadHandler.java | 43 +
.../handler/LogviewerLogPageHandler.java | 410 ++++++
.../handler/LogviewerLogSearchHandler.java | 707 ++++++++++
.../handler/LogviewerProfileHandler.java | 115 ++
.../logviewer/utils/DirectoryCleaner.java | 183 +++
.../daemon/logviewer/utils/LogCleaner.java | 305 +++++
.../logviewer/utils/LogFileDownloader.java | 51 +
.../utils/LogviewerResponseBuilder.java | 118 ++
.../logviewer/utils/ResourceAuthorizer.java | 130 ++
.../daemon/logviewer/utils/WorkerLogs.java | 60 +
.../logviewer/webapp/LogviewerApplication.java | 94 ++
.../logviewer/webapp/LogviewerResource.java | 221 +++
.../daemon/wip/logviewer/LogviewerConstant.java | 23 -
.../daemon/wip/logviewer/LogviewerServer.java | 174 ---
.../handler/LogviewerLogDownloadHandler.java | 43 -
.../handler/LogviewerLogPageHandler.java | 412 ------
.../handler/LogviewerLogSearchHandler.java | 686 ----------
.../handler/LogviewerProfileHandler.java | 115 --
.../daemon/wip/logviewer/utils/LogCleaner.java | 296 -----
.../wip/logviewer/utils/LogFileDownloader.java | 51 -
.../utils/LogviewerResponseBuilder.java | 118 --
.../wip/logviewer/utils/ResourceAuthorizer.java | 129 --
.../daemon/wip/logviewer/utils/WorkerLogs.java | 63 -
.../logviewer/webapp/LogviewerApplication.java | 94 --
.../wip/logviewer/webapp/LogviewerResource.java | 221 ---
.../storm/daemon/logviewer/LogviewerTest.java | 51 +
.../handler/LogviewerLogPageHandlerTest.java | 101 ++
.../handler/LogviewerLogSearchHandlerTest.java | 854 ++++++++++++
.../testsupport/ArgumentsVerifier.java | 34 +
.../testsupport/MockDirectoryBuilder.java | 66 +
.../logviewer/testsupport/MockFileBuilder.java | 66 +
.../daemon/logviewer/utils/LogCleanerTest.java | 376 ++++++
.../logviewer/utils/ResourceAuthorizerTest.java | 182 +++
.../logviewer-search-context-tests.log.gz | Bin 0 -> 72 bytes
.../logviewer-search-context-tests.log.test | 1 +
.../src/test/resources/small-worker.log.test | 1 +
.../src/test/resources/test-3072.log.test | 3 +
.../src/test/resources/test-worker.log.test | 380 ++++++
49 files changed, 4748 insertions(+), 5077 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index dade6b5..a3c6506 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -812,7 +812,7 @@ def logviewer():
allextrajars = get_wildcard_dir(STORM_WEBAPP_LIB_DIR)
allextrajars.append(CLUSTER_CONF_DIR)
exec_storm_class(
- "org.apache.storm.daemon.wip.logviewer.LogviewerServer",
+ "org.apache.storm.daemon.logviewer.LogviewerServer",
jvmtype="-server",
daemonName="logviewer",
jvmopts=jvmopts,
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
deleted file mode 100644
index 27b4ba1..0000000
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ /dev/null
@@ -1,1254 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.daemon.logviewer
- (:use compojure.core)
- (:use [clojure.set :only [difference intersection]])
- (:use [clojure.string :only [blank? split]])
- (:use [hiccup core page-helpers form-helpers])
- (:use [org.apache.storm config daemon-config util log])
- (:use [org.apache.storm.ui helpers])
- (:import [org.apache.storm StormTimer]
- [org.apache.storm.daemon.supervisor ClientSupervisorUtils]
- [org.apache.storm.daemon.supervisor SupervisorUtils]
- [org.apache.storm.metric StormMetricsRegistry])
- (:import [org.apache.storm.utils Time VersionInfo ConfigUtils Utils ServerUtils ServerConfigUtils])
- (:import [java.util Arrays ArrayList HashSet])
- (:import [java.util.zip GZIPInputStream])
- (:import [org.apache.logging.log4j LogManager])
- (:import [org.apache.logging.log4j.core.appender RollingFileAppender])
- (:import [java.io BufferedInputStream File FileFilter FileInputStream
- InputStream]
- [java.net URLDecoder])
- (:import [java.nio.file Files DirectoryStream])
- (:import [java.nio ByteBuffer])
- (:import [org.apache.storm.daemon DirectoryCleaner])
- (:import [org.apache.storm.ui InvalidRequestException UIHelpers IConfigurator FilterConfiguration]
- [org.apache.storm.security.auth AuthUtils])
- (:require [compojure.route :as route]
- [compojure.handler :as handler]
- [ring.middleware.keyword-params]
- [ring.util.codec :as codec]
- [ring.util.response :as resp]
- [clojure.string :as string])
- (:gen-class))
-
-(def ^:dynamic *STORM-CONF* (clojurify-structure (ConfigUtils/readStormConfig)))
-(def STORM-VERSION (VersionInfo/getVersion))
-
-(def worker-log-filename-pattern #"^worker.log(.*)")
-
-(def logviewer:num-log-page-http-requests (StormMetricsRegistry/registerMeter "logviewer:num-log-page-http-requests"))
-(def logviewer:num-daemonlog-page-http-requests (StormMetricsRegistry/registerMeter "logviewer:num-daemonlog-page-http-requests"))
-(def logviewer:num-download-log-file-http-requests (StormMetricsRegistry/registerMeter "logviewer:num-download-log-file-http-requests"))
-(def logviewer:num-download-log-daemon-file-http-requests (StormMetricsRegistry/registerMeter "logviewer:num-download-log-daemon-file-http-requests"))
-(def logviewer:num-list-logs-http-requests (StormMetricsRegistry/registerMeter "logviewer:num-list-logs-http-requests"))
-
-(defn cleanup-cutoff-age-millis [conf now-millis]
- (- now-millis (* (conf LOGVIEWER-CLEANUP-AGE-MINS) 60 1000)))
-
-(defn get-stream-for-dir
- [^File f]
- (try (Files/newDirectoryStream (.toPath f))
- (catch Exception ex (log-error ex) nil)))
-
-(defn- last-modifiedtime-worker-logdir
- "Return the last modified time for all log files in a worker's log dir.
- Using stream rather than File.listFiles is to avoid large mem usage
- when a directory has too many files"
- [^File log-dir]
- (let [^DirectoryStream stream (get-stream-for-dir log-dir)
- dir-modified (.lastModified log-dir)
- last-modified (try (reduce
- (fn [maximum path]
- (let [curr (.lastModified (.toFile path))]
- (if (> curr maximum)
- curr
- maximum)))
- dir-modified
- stream)
- (catch Exception ex
- (log-error ex) dir-modified)
- (finally
- (if (instance? DirectoryStream stream)
- (.close stream))))]
- last-modified))
-
-(defn get-size-for-logdir
- "Return the sum of lengths for all log files in a worker's log dir.
- Using stream rather than File.listFiles is to avoid large mem usage
- when a directory has too many files"
- [log-dir]
- (let [^DirectoryStream stream (get-stream-for-dir log-dir)]
- (reduce
- (fn [sum path]
- (let [size (.length (.toFile path))]
- (+ sum size)))
- 0
- stream)))
-
-(defn mk-FileFilter-for-log-cleanup [conf now-millis]
- (let [cutoff-age-millis (cleanup-cutoff-age-millis conf now-millis)]
- (reify FileFilter (^boolean accept [this ^File file]
- (boolean (and
- (not (.isFile file))
- (<= (last-modifiedtime-worker-logdir file) cutoff-age-millis)))))))
-
-(defn select-dirs-for-cleanup [conf now-millis root-dir]
- (let [file-filter (mk-FileFilter-for-log-cleanup conf now-millis)]
- (reduce clojure.set/union
- (sorted-set)
- (for [^File topo-dir (.listFiles (File. root-dir))]
- (into [] (.listFiles topo-dir file-filter))))))
-
-(defn get-topo-port-workerlog
- "Return the path of the worker log with the format of topoId/port/worker.log.*"
- [^File file]
- (clojure.string/join ServerUtils/FILE_PATH_SEPARATOR
- (take-last 3
- (split (.getCanonicalPath file) (re-pattern ServerUtils/FILE_PATH_SEPARATOR)))))
-
-(defn get-metadata-file-for-log-root-name [root-name root-dir]
- (let [metaFile (clojure.java.io/file root-dir "metadata"
- (str root-name ".yaml"))]
- (if (.exists metaFile)
- metaFile
- (do
- (log-warn "Could not find " (.getCanonicalPath metaFile)
- " to clean up for " root-name)
- nil))))
-
-(defn get-metadata-file-for-wroker-logdir [logdir]
- (let [metaFile (clojure.java.io/file logdir "worker.yaml")]
- (if (.exists metaFile)
- metaFile
- (do
- (log-warn "Could not find " (.getCanonicalPath metaFile)
- " to clean up for " logdir)
- nil))))
-
-(defn get-worker-id-from-metadata-file [metaFile]
- (get (clojurify-structure (Utils/readYamlFile metaFile)) "worker-id"))
-
-(defn get-topo-owner-from-metadata-file [metaFile]
- (get (clojurify-structure (Utils/readYamlFile metaFile)) TOPOLOGY-SUBMITTER-USER))
-
-(defn identify-worker-log-dirs [log-dirs]
- "return the workerid to worker-log-dir map"
- (into {} (for [logdir log-dirs
- :let [metaFile (get-metadata-file-for-wroker-logdir logdir)]]
- (if metaFile
- {(get-worker-id-from-metadata-file metaFile) logdir}
- {"" logdir})))) ;; an old directory that has no yaml file will be treated as a dead dir for deleting
-
-(defn get-alive-ids
- [conf now-secs]
- (->>
- (clojurify-structure (SupervisorUtils/readWorkerHeartbeats conf))
- (remove
- #(or (not (val %))
- (SupervisorUtils/isWorkerHbTimedOut now-secs
- (val %)
- conf)))
- keys
- set))
-
-(defn get-dead-worker-dirs
- "Return a sorted set of java.io.Files that were written by workers that are
- now dead"
- [conf now-secs log-dirs]
- (if (empty? log-dirs)
- (sorted-set)
- (let [alive-ids (get-alive-ids conf now-secs)
- id->dir (identify-worker-log-dirs log-dirs)]
- (apply sorted-set
- (for [[id dir] id->dir
- :when (not (contains? alive-ids id))]
- dir)))))
-
-(defn get-all-worker-dirs [^File root-dir]
- (reduce clojure.set/union
- (sorted-set)
- (for [^File topo-dir (.listFiles root-dir)]
- (into [] (.listFiles topo-dir)))))
-
-(defn get-alive-worker-dirs
- "Return a sorted set of java.io.Files that were written by workers that are
- now active"
- [conf root-dir]
- (let [alive-ids (get-alive-ids conf (Time/currentTimeSecs))
- log-dirs (get-all-worker-dirs root-dir)
- id->dir (identify-worker-log-dirs log-dirs)]
- (apply sorted-set
- (for [[id dir] id->dir
- :when (contains? alive-ids id)]
- (.getCanonicalPath dir)))))
-
-(defn get-all-logs-for-rootdir [^File log-dir]
- (reduce concat
- (for [port-dir (get-all-worker-dirs log-dir)]
- (into [] (DirectoryCleaner/getFilesForDir port-dir)))))
-
-(defn is-active-log [^File file]
- (re-find #"\.(log|err|out|current|yaml|pid)$" (.getName file)))
-
-(defn sum-file-size
- "Given a sequence of Files, sum their sizes."
- [files]
- (reduce #(+ %1 (.length %2)) 0 files))
-
-(defn per-workerdir-cleanup!
- "Delete the oldest files in each overloaded worker log dir"
- [^File root-dir size ^DirectoryCleaner cleaner]
- (dofor [worker-dir (get-all-worker-dirs root-dir)]
- (.deleteOldestWhileTooLarge cleaner (ArrayList. [worker-dir]) size true nil)))
-
-(defn global-log-cleanup!
- "Delete the oldest files in overloaded worker-artifacts globally"
- [^File root-dir size ^DirectoryCleaner cleaner]
- (let [worker-dirs (ArrayList. (get-all-worker-dirs root-dir))
- alive-worker-dirs (HashSet. (get-alive-worker-dirs *STORM-CONF* root-dir))]
- (.deleteOldestWhileTooLarge cleaner worker-dirs size false alive-worker-dirs)))
-
-(defn cleanup-empty-topodir!
- "Delete the topo dir if it contains zero port dirs"
- [^File dir]
- (let [topodir (.getParentFile dir)]
- (if (empty? (.listFiles topodir))
- (Utils/forceDelete (.getCanonicalPath topodir)))))
-
-(defn cleanup-fn!
- "Delete old log dirs for which the workers are no longer alive"
- [log-root-dir]
- (let [now-secs (Time/currentTimeSecs)
- old-log-dirs (select-dirs-for-cleanup *STORM-CONF*
- (* now-secs 1000)
- log-root-dir)
- total-size (*STORM-CONF* LOGVIEWER-MAX-SUM-WORKER-LOGS-SIZE-MB)
- per-dir-size (*STORM-CONF* LOGVIEWER-MAX-PER-WORKER-LOGS-SIZE-MB)
- per-dir-size (min per-dir-size (* total-size 0.5))
- cleaner (DirectoryCleaner.)
- dead-worker-dirs (get-dead-worker-dirs *STORM-CONF*
- now-secs
- old-log-dirs)]
- (log-debug "log cleanup: now=" now-secs
- " old log dirs " (pr-str (map #(.getName %) old-log-dirs))
- " dead worker dirs " (pr-str
- (map #(.getName %) dead-worker-dirs)))
- (dofor [dir dead-worker-dirs]
- (let [path (.getCanonicalPath dir)]
- (log-message "Cleaning up: Removing " path)
- (try (Utils/forceDelete path)
- (cleanup-empty-topodir! dir)
- (catch Exception ex (log-error ex)))))
- (per-workerdir-cleanup! (File. log-root-dir) (* per-dir-size (* 1024 1024)) cleaner)
- (let [size (* total-size (* 1024 1024))]
- (global-log-cleanup! (File. log-root-dir) size cleaner))))
-
-(defn start-log-cleaner! [conf log-root-dir]
- (let [interval-secs (conf LOGVIEWER-CLEANUP-INTERVAL-SECS)]
- (when interval-secs
- (log-debug "starting log cleanup thread at interval: " interval-secs)
-
- (let [timer (StormTimer. "logviewer-cleanup"
- (reify Thread$UncaughtExceptionHandler
- (^void uncaughtException
- [this ^Thread t ^Throwable e]
- (log-error t "Error when doing logs cleanup")
- (Utils/exitProcess 20 "Error when doing log cleanup"))))]
- (.scheduleRecurring timer 0 interval-secs
- (fn [] (cleanup-fn! log-root-dir)))))))
-
-(defn- skip-bytes
- "FileInputStream#skip may not work the first time, so ensure it successfully
- skips the given number of bytes."
- [^InputStream stream n]
- (loop [skipped 0]
- (let [skipped (+ skipped (.skip stream (- n skipped)))]
- (if (< skipped n) (recur skipped)))))
-
-(defn logfile-matches-filter?
- [log-file-name]
- (let [regex-string (str "worker.log.*")
- regex-pattern (re-pattern regex-string)]
- (not= (re-seq regex-pattern (.toString log-file-name)) nil)))
-
-(defn page-file
- ([path tail]
- (let [zip-file? (.endsWith path ".gz")
- flen (if zip-file? (ServerUtils/zipFileSize (clojure.java.io/file path)) (.length (clojure.java.io/file path)))
- skip (- flen tail)]
- (page-file path skip tail)))
- ([path start length]
- (let [zip-file? (.endsWith path ".gz")
- flen (if zip-file? (ServerUtils/zipFileSize (clojure.java.io/file path)) (.length (clojure.java.io/file path)))]
- (with-open [input (if zip-file? (GZIPInputStream. (FileInputStream. path)) (FileInputStream. path))
- output (java.io.ByteArrayOutputStream.)]
- (if (>= start flen)
- (throw
- (InvalidRequestException. "Cannot start past the end of the file")))
- (if (> start 0) (skip-bytes input start))
- (let [buffer (make-array Byte/TYPE 1024)]
- (loop []
- (when (< (.size output) length)
- (let [size (.read input buffer 0 (min 1024 (- length (.size output))))]
- (when (pos? size)
- (.write output buffer 0 size)
- (recur)))))
- (.toString output))))))
-
-(defn get-log-user-group-whitelist [fname]
- (let [wl-file (ServerConfigUtils/getLogMetaDataFile fname)
- m (clojurify-structure (Utils/readYamlFile wl-file))]
- (if (not-nil? m)
- (do
- (let [user-wl (.get m LOGS-USERS)
- user-wl (if user-wl user-wl [])
- group-wl (.get m LOGS-GROUPS)
- group-wl (if group-wl group-wl [])]
- [user-wl group-wl]))
- nil)))
-
-(def igroup-mapper (AuthUtils/GetGroupMappingServiceProviderPlugin *STORM-CONF*))
-(defn user-groups
- [user]
- (if (blank? user) [] (.getGroups igroup-mapper user)))
-
-(defn authorized-log-user? [user fname conf]
- (if (or (blank? user) (blank? fname) (nil? (get-log-user-group-whitelist fname)))
- nil
- (let [groups (user-groups user)
- [user-wl group-wl] (get-log-user-group-whitelist fname)
- logs-users (concat (conf LOGS-USERS)
- (conf NIMBUS-ADMINS)
- user-wl)
- logs-groups (concat (conf LOGS-GROUPS)
- group-wl)]
- (or (some #(= % user) logs-users)
- (< 0 (.size (intersection (set groups) (set logs-groups))))))))
-
-(defn log-root-dir
- "Given an appender name, as configured, get the parent directory of the appender's log file.
- Note that if anything goes wrong, this will throw an Error and exit."
- [appender-name]
- (let [appender (.getAppender (.getConfiguration (LogManager/getContext)) appender-name)]
- (if (and appender-name appender (instance? RollingFileAppender appender))
- (.getParent (File. (.getFileName appender)))
- (throw
- (RuntimeException. "Log viewer could not find configured appender, or the appender is not a FileAppender. Please check that the appender name configured in storm and log4j agree.")))))
-
-(defnk to-btn-link
- "Create a link that is formatted like a button"
- [url text :enabled true]
- [:a {:href (java.net.URI. url)
- :class (str "btn btn-default " (if enabled "enabled" "disabled"))} text])
-
-(defn search-file-form [fname is-daemon]
- [[:form {:action "logviewer_search.html" :id "search-box"}
- "Search this file:"
- [:input {:type "text" :name "search"}]
- [:input {:type "hidden" :name "is-daemon" :value is-daemon}]
- [:input {:type "hidden" :name "file" :value fname}]
- [:input {:type "submit" :value "Search"}]]])
-
-(defn log-file-selection-form [log-files type]
- [[:form {:action type :id "list-of-files"}
- (drop-down "file" log-files)
- [:input {:type "submit" :value "Switch file"}]]])
-
-(defn pager-links [fname start length file-size type]
- (let [prev-start (max 0 (- start length))
- next-start (if (> file-size 0)
- (min (max 0 (- file-size length)) (+ start length))
- (+ start length))]
- [[:div
- (concat
- [(to-btn-link (url (str "/" type)
- {:file fname
- :start (max 0 (- start length))
- :length length})
- "Prev" :enabled (< prev-start start))]
- [(to-btn-link (url (str "/" type)
- {:file fname
- :start 0
- :length length}) "First")]
- [(to-btn-link (url (str "/" type)
- {:file fname
- :length length})
- "Last")]
- [(to-btn-link (url (str "/" type)
- {:file fname
- :start (min (max 0 (- file-size length))
- (+ start length))
- :length length})
- "Next" :enabled (> next-start start))])]]))
-
-(defn- download-link [fname]
- [[:p (link-to (UIHelpers/urlFormat "/download?file=%s" (to-array [fname])) "Download Full File")]])
-
-(defn- daemon-download-link [fname]
- [[:p (link-to (UIHelpers/urlFormat "/daemondownload/%s" (to-array [fname])) "Download Full File")]])
-
-(defn- is-txt-file [fname]
- (re-find #"\.(log.*|txt|yaml|pid)$" fname))
-
-(defn unauthorized-user-html [user]
- [[:h2 "User '" (escape-html user) "' is not authorized."]])
-
-(defn ring-response-from-exception [ex]
- {:headers {}
- :status 400
- :body (.getMessage ex)})
-
-(def default-bytes-per-page 51200)
-
-(defn log-page [fname start length grep user root-dir]
- (if (or (blank? (*STORM-CONF* UI-FILTER))
- (authorized-log-user? user fname *STORM-CONF*))
- (let [file (.getCanonicalFile (File. root-dir fname))
- path (.getCanonicalPath file)
- zip-file? (.endsWith path ".gz")
- topo-dir (.getParentFile (.getParentFile file))]
- (if (and (.exists file)
- (= (.getCanonicalFile (File. root-dir))
- (.getParentFile topo-dir)))
- (let [file-length (if zip-file? (ServerUtils/zipFileSize (clojure.java.io/file path)) (.length (clojure.java.io/file path)))
- log-files (reduce clojure.set/union
- (sorted-set)
- (for [^File port-dir (.listFiles topo-dir)]
- (into [] (filter #(.isFile %) (DirectoryCleaner/getFilesForDir port-dir))))) ;all types of files included
- files-str (for [file log-files]
- (get-topo-port-workerlog file))
- reordered-files-str (conj (filter #(not= fname %) files-str) fname)
- length (if length
- (min 10485760 length)
- default-bytes-per-page)
- log-string (escape-html
- (if (is-txt-file fname)
- (if start
- (page-file path start length)
- (page-file path length))
- "This is a binary file and cannot display! You may download the full file."))
- start (or start (- file-length length))]
- (if grep
- (html [:pre#logContent
- (if grep
- (->> (.split log-string "\n")
- (filter #(.contains % grep))
- (string/join "\n"))
- log-string)])
- (let [pager-data (if (is-txt-file fname) (pager-links fname start length file-length "log") nil)]
- (html (concat (search-file-form fname "no")
- (log-file-selection-form reordered-files-str "log") ; list all files for this topology
- pager-data
- (download-link fname)
- [[:pre#logContent log-string]]
- pager-data)))))
- (-> (resp/response "Page not found")
- (resp/status 404))))
- (if (nil? (get-log-user-group-whitelist fname))
- (-> (resp/response "Page not found")
- (resp/status 404))
- (unauthorized-user-html user))))
-
-(defn daemonlog-page [fname start length grep user root-dir]
- (let [file (.getCanonicalFile (File. root-dir fname))
- file-length (.length file)
- path (.getCanonicalPath file)
- zip-file? (.endsWith path ".gz")]
- (if (and (= (.getCanonicalFile (File. root-dir))
- (.getParentFile file))
- (.exists file))
- (let [file-length (if zip-file? (ServerUtils/zipFileSize (clojure.java.io/file path)) (.length (clojure.java.io/file path)))
- length (if length
- (min 10485760 length)
- default-bytes-per-page)
- log-files (into [] (filter #(.isFile %) (.listFiles (File. root-dir)))) ;all types of files included
- files-str (for [file log-files]
- (.getName file))
- reordered-files-str (conj (filter #(not= fname %) files-str) fname)
- log-string (escape-html
- (if (is-txt-file fname)
- (if start
- (page-file path start length)
- (page-file path length))
- "This is a binary file and cannot display! You may download the full file."))
- start (or start (- file-length length))]
- (if grep
- (html [:pre#logContent
- (if grep
- (->> (.split log-string "\n")
- (filter #(.contains % grep))
- (string/join "\n"))
- log-string)])
- (let [pager-data (if (is-txt-file fname) (pager-links fname start length file-length "daemonlog") nil)]
- (html (concat (search-file-form fname "yes")
- (log-file-selection-form reordered-files-str "daemonlog") ; list all daemon logs
- pager-data
- (daemon-download-link fname)
- [[:pre#logContent log-string]]
- pager-data)))))
- (-> (resp/response "Page not found")
- (resp/status 404)))))
-
-(defnk set-log-file-permissions [fname root-dir]
- (let [file (.getCanonicalFile (File. root-dir fname))
- run-as-user (*STORM-CONF* SUPERVISOR-RUN-WORKER-AS-USER)
- parent (.getParent (File. root-dir fname))
- md-file (if (nil? parent) nil (get-metadata-file-for-wroker-logdir parent))
- topo-owner (if (nil? md-file) nil (get-topo-owner-from-metadata-file (.getCanonicalPath md-file)))]
- (when (and run-as-user
- (not-nil? topo-owner)
- (.exists file)
- (not (Files/isReadable (.toPath file))))
- (log-debug "Setting permissions on file " fname " with topo-owner " topo-owner)
- (ClientSupervisorUtils/processLauncherAndWait *STORM-CONF* topo-owner ["blob" (.getCanonicalPath file)] nil (str "setup group read permissions for file: " fname)))))
-
-(defnk download-log-file [fname req resp user ^String root-dir :is-daemon false]
- (let [file (.getCanonicalFile (File. root-dir fname))]
- (if (.exists file)
-
- (if (or is-daemon
- (or (blank? (*STORM-CONF* UI-FILTER))
- (authorized-log-user? user fname *STORM-CONF*)))
- (-> (resp/response file)
- (resp/content-type "application/octet-stream"))
- (unauthorized-user-html user))
- (-> (resp/response "Page not found")
- (resp/status 404)))))
-
-(def grep-max-search-size 1024)
-(def grep-buf-size 2048)
-(def grep-context-size 128)
-
-(defn logviewer-port
- []
- (int (*STORM-CONF* LOGVIEWER-PORT)))
-
-(defn url-to-match-centered-in-log-page
- [needle fname offset port]
- (let [host (Utils/hostname)
- port (logviewer-port)
- fname (clojure.string/join ServerUtils/FILE_PATH_SEPARATOR (take-last 3 (split fname (re-pattern ServerUtils/FILE_PATH_SEPARATOR))))]
- (url (str "http://" host ":" port "/log")
- {:file fname
- :start (max 0
- (- offset
- (int (/ default-bytes-per-page 2))
- (int (/ (alength needle) -2)))) ;; Addition
- :length default-bytes-per-page})))
-
-(defn url-to-match-centered-in-log-page-daemon-file
- [needle fname offset port]
- (let [host (Utils/hostname)
- port (logviewer-port)
- fname (clojure.string/join ServerUtils/FILE_PATH_SEPARATOR (take-last 1 (split fname (re-pattern ServerUtils/FILE_PATH_SEPARATOR))))]
- (url (str "http://" host ":" port "/daemonlog")
- {:file fname
- :start (max 0
- (- offset
- (int (/ default-bytes-per-page 2))
- (int (/ (alength needle) -2)))) ;; Addition
- :length default-bytes-per-page})))
-
-(defnk mk-match-data
- [^bytes needle ^ByteBuffer haystack haystack-offset file-offset fname
- :is-daemon false :before-bytes nil :after-bytes nil]
- (let [url (if is-daemon
- (url-to-match-centered-in-log-page-daemon-file needle
- fname
- file-offset
- (*STORM-CONF* LOGVIEWER-PORT))
- (url-to-match-centered-in-log-page needle
- fname
- file-offset
- (*STORM-CONF* LOGVIEWER-PORT)))
- haystack-bytes (.array haystack)
- before-string (if (>= haystack-offset grep-context-size)
- (String. haystack-bytes
- (- haystack-offset grep-context-size)
- grep-context-size
- "UTF-8")
- (let [num-desired (max 0 (- grep-context-size
- haystack-offset))
- before-size (if before-bytes
- (alength before-bytes)
- 0)
- num-expected (min before-size num-desired)]
- (if (pos? num-expected)
- (str (String. before-bytes
- (- before-size num-expected)
- num-expected
- "UTF-8")
- (String. haystack-bytes
- 0
- haystack-offset
- "UTF-8"))
- (String. haystack-bytes
- 0
- haystack-offset
- "UTF-8"))))
- after-string (let [needle-size (alength needle)
- after-offset (+ haystack-offset needle-size)
- haystack-size (.limit haystack)]
- (if (< (+ after-offset grep-context-size) haystack-size)
- (String. haystack-bytes
- after-offset
- grep-context-size
- "UTF-8")
- (let [num-desired (- grep-context-size
- (- haystack-size after-offset))
- after-size (if after-bytes
- (alength after-bytes)
- 0)
- num-expected (min after-size num-desired)]
- (if (pos? num-expected)
- (str (String. haystack-bytes
- after-offset
- (- haystack-size after-offset)
- "UTF-8")
- (String. after-bytes 0 num-expected "UTF-8"))
- (String. haystack-bytes
- after-offset
- (- haystack-size after-offset)
- "UTF-8")))))]
- {"byteOffset" file-offset
- "beforeString" before-string
- "afterString" after-string
- "matchString" (String. needle "UTF-8")
- "logviewerURL" url}))
-
-(defn- try-read-ahead!
- "Tries once to read ahead in the stream to fill the context and resets the
- stream to its position before the call."
- [^BufferedInputStream stream haystack offset file-len bytes-read]
- (let [num-expected (min (- file-len bytes-read)
- grep-context-size)
- after-bytes (byte-array num-expected)]
- (.mark stream num-expected)
- ;; Only try reading once.
- (.read stream after-bytes 0 num-expected)
- (.reset stream)
- after-bytes))
-
-(defn offset-of-bytes
- "Searches a given byte array for a match of a sub-array of bytes. Returns
- the offset to the byte that matches, or -1 if no match was found."
- [^bytes buf ^bytes value init-offset]
- {:pre [(> (alength value) 0)
- (not (neg? init-offset))]}
- (loop [offset init-offset
- candidate-offset init-offset
- val-offset 0]
- (if-not (pos? (- (alength value) val-offset))
- ;; Found
- candidate-offset
- (if (>= offset (alength buf))
- ;; We ran out of buffer for the search.
- -1
- (if (not= (aget value val-offset) (aget buf offset))
- ;; The match at this candidate offset failed, so start over with the
- ;; next candidate byte from the buffer.
- (let [new-offset (inc candidate-offset)]
- (recur new-offset new-offset 0))
- ;; So far it matches. Keep going...
- (recur (inc offset) candidate-offset (inc val-offset)))))))
-
-(defn- buffer-substring-search!
- "As the file is read into a buffer, 1/2 the buffer's size at a time, we
- search the buffer for matches of the substring and return a list of zero or
- more matches."
- [is-daemon file file-len offset-to-buf init-buf-offset stream bytes-skipped
- bytes-read ^ByteBuffer haystack ^bytes needle initial-matches num-matches
- ^bytes before-bytes]
- (loop [buf-offset init-buf-offset
- matches initial-matches]
- (let [offset (offset-of-bytes (.array haystack) needle buf-offset)]
- (if (and (< (count matches) num-matches) (not (neg? offset)))
- (let [file-offset (+ offset-to-buf offset)
- bytes-needed-after-match (- (.limit haystack)
- grep-context-size
- (alength needle))
- before-arg (if (< offset grep-context-size) before-bytes)
- after-arg (if (> offset bytes-needed-after-match)
- (try-read-ahead! stream
- haystack
- offset
- file-len
- bytes-read))]
- (recur (+ offset (alength needle))
- (conj matches
- (mk-match-data needle
- haystack
- offset
- file-offset
- (.getCanonicalPath file)
- :is-daemon is-daemon
- :before-bytes before-arg
- :after-bytes after-arg))))
- (let [before-str-to-offset (min (.limit haystack)
- grep-max-search-size)
- before-str-from-offset (max 0 (- before-str-to-offset
- grep-context-size))
- new-before-bytes (Arrays/copyOfRange (.array haystack)
- before-str-from-offset
- before-str-to-offset)
- ;; It's OK if new-byte-offset is negative. This is normal if
- ;; we are out of bytes to read from a small file.
- new-byte-offset (if (>= (count matches) num-matches)
- (+ (get (last matches) "byteOffset")
- (alength needle))
- (+ bytes-skipped
- bytes-read
- (- grep-max-search-size)))]
- [matches new-byte-offset new-before-bytes])))))
-
-(defn- mk-grep-response
- "This response data only includes a next byte offset if there is more of the
- file to read."
- [search-bytes offset matches next-byte-offset]
- (merge {"searchString" (String. search-bytes "UTF-8")
- "startByteOffset" offset
- "matches" matches}
- (and next-byte-offset {"nextByteOffset" next-byte-offset})))
-
-(defn rotate-grep-buffer!
- [^ByteBuffer buf ^BufferedInputStream stream total-bytes-read file file-len]
- (let [buf-arr (.array buf)]
- ;; Copy the 2nd half of the buffer to the first half.
- (System/arraycopy buf-arr
- grep-max-search-size
- buf-arr
- 0
- grep-max-search-size)
- ;; Zero-out the 2nd half to prevent accidental matches.
- (Arrays/fill buf-arr
- grep-max-search-size
- (count buf-arr)
- (byte 0))
- ;; Fill the 2nd half with new bytes from the stream.
- (let [bytes-read (.read stream
- buf-arr
- grep-max-search-size
- (min file-len grep-max-search-size))]
- (.limit buf (+ grep-max-search-size bytes-read))
- (swap! total-bytes-read + bytes-read))))
-
-(defnk substring-search
- "Searches for a substring in a log file, starting at the given offset,
- returning the given number of matches, surrounded by the given number of
- context lines. Other information is included to be useful for progressively
- searching through a file for display in a UI. The search string must
- grep-max-search-size bytes or fewer when decoded with UTF-8."
- [file ^String search-string :is-daemon false :num-matches 10 :start-byte-offset 0]
- {:pre [(not (empty? search-string))
- (<= (count (.getBytes search-string "UTF-8")) grep-max-search-size)]}
- (let [zip-file? (.endsWith (.getName file) ".gz")
- f-input-steam (FileInputStream. file)
- gzipped-input-stream (if zip-file?
- (GZIPInputStream. f-input-steam)
- f-input-steam)
- stream ^BufferedInputStream (BufferedInputStream.
- gzipped-input-stream)
- file-len (if zip-file? (ServerUtils/zipFileSize file) (.length file))
- buf ^ByteBuffer (ByteBuffer/allocate grep-buf-size)
- buf-arr ^bytes (.array buf)
- string nil
- total-bytes-read (atom 0)
- matches []
- search-bytes ^bytes (.getBytes search-string "UTF-8")
- num-matches (or num-matches 10)
- start-byte-offset (or start-byte-offset 0)]
- ;; Start at the part of the log file we are interested in.
- ;; Allow searching when start-byte-offset == file-len so it doesn't blow up on 0-length files
- (if (> start-byte-offset file-len)
- (throw
- (InvalidRequestException. "Cannot search past the end of the file")))
- (when (> start-byte-offset 0)
- (skip-bytes stream start-byte-offset))
- (java.util.Arrays/fill buf-arr (byte 0))
- (let [bytes-read (.read stream buf-arr 0 (min file-len grep-buf-size))]
- (.limit buf bytes-read)
- (swap! total-bytes-read + bytes-read))
- (loop [initial-matches []
- init-buf-offset 0
- byte-offset start-byte-offset
- before-bytes nil]
- (let [[matches new-byte-offset new-before-bytes]
- (buffer-substring-search!
- is-daemon
- file
- file-len
- byte-offset
- init-buf-offset
- stream
- start-byte-offset
- @total-bytes-read
- buf
- search-bytes
- initial-matches
- num-matches
- before-bytes)]
- (if (and (< (count matches) num-matches)
- (< (+ @total-bytes-read start-byte-offset) file-len))
- (let [;; The start index is positioned to find any possible
- ;; occurrence search string that did not quite fit in the
- ;; buffer on the previous read.
- new-buf-offset (- (min (.limit ^ByteBuffer buf)
- grep-max-search-size)
- (alength search-bytes))]
- (rotate-grep-buffer! buf stream total-bytes-read file file-len)
- (when (< @total-bytes-read 0)
- (throw (InvalidRequestException. "Cannot search past the end of the file")))
- (recur matches
- new-buf-offset
- new-byte-offset
- new-before-bytes))
- (merge {"isDaemon" (if is-daemon "yes" "no")}
- (mk-grep-response search-bytes
- start-byte-offset
- matches
- (if-not (and (< (count matches) num-matches)
- (>= @total-bytes-read file-len))
- (let [next-byte-offset (+ (get (last matches)
- "byteOffset")
- (alength search-bytes))]
- (if (> file-len next-byte-offset)
- next-byte-offset))))))))))
-
-(defn- try-parse-int-param
- [nam value]
- (try
- (Integer/parseInt value)
- (catch java.lang.NumberFormatException e
- (->
- (str "Could not parse " nam " to an integer")
- (InvalidRequestException. e)
- throw))))
-
-(defn search-log-file
- [fname user ^String root-dir is-daemon search num-matches offset callback origin]
- (let [file (.getCanonicalFile (File. root-dir fname))]
- (if (.exists file)
- (if (or is-daemon
- (or (blank? (*STORM-CONF* UI-FILTER))
- (authorized-log-user? user fname *STORM-CONF*)))
- (let [num-matches-int (if num-matches
- (try-parse-int-param "num-matches"
- num-matches))
- offset-int (if offset
- (try-parse-int-param "start-byte-offset" offset))]
- (try
- (if (and (not (empty? search))
- <= (count (.getBytes search "UTF-8")) grep-max-search-size)
- (json-response
- (merge {"isDaemon" (if is-daemon "yes" "no")}
- (substring-search file
- search
- :is-daemon is-daemon
- :num-matches num-matches-int
- :start-byte-offset offset-int))
-
- callback
- :headers {"Access-Control-Allow-Origin" origin
- "Access-Control-Allow-Credentials" "true"})
- (throw
- (InvalidRequestException.
- (str "Search substring must be between 1 and 1024 UTF-8 "
- "bytes in size (inclusive)"))))
- (catch Exception ex
- (json-response (UIHelpers/exceptionToJson ex) callback :status 500))))
- (json-response (UIHelpers/unauthorizedUserJson user) callback :status 401))
- (json-response {"error" "Not Found"
- "errorMessage" "The file was not found on this node."}
- callback
- :status 404))))
-
-(defn find-n-matches [logs n file-offset offset search]
- (let [logs (drop file-offset logs)
- wrap-matches-fn (fn [matches]
- {"fileOffset" file-offset
- "searchString" search
- "matches" matches})]
- (loop [matches []
- logs logs
- offset offset
- file-offset file-offset
- match-count 0]
- (if (empty? logs)
- (wrap-matches-fn matches)
- (let [these-matches (try
- (log-debug "Looking through " (first logs))
- (substring-search (first logs)
- search
- :num-matches (- n match-count)
- :start-byte-offset offset)
- (catch InvalidRequestException e
- (log-error e "Can't search past end of file.")
- {}))
- file-name (get-topo-port-workerlog (first logs))
- new-matches (conj matches
- (merge these-matches
- { "fileName" file-name
- "port" (first (take-last 2 (split (.getCanonicalPath (first logs)) (re-pattern ServerUtils/FILE_PATH_SEPARATOR))))}))
- new-count (+ match-count (count (these-matches "matches")))]
- (if (empty? these-matches)
- (recur matches (rest logs) 0 (+ file-offset 1) match-count)
- (if (>= new-count n)
- (wrap-matches-fn new-matches)
- (recur new-matches (rest logs) 0 (+ file-offset 1) new-count))))))))
-
-(defn logs-for-port
- "Get the filtered, authorized, sorted log files for a port."
- [user port-dir]
- (let [filter-authorized-fn (fn [user logs]
- (filter #(or
- (blank? (*STORM-CONF* UI-FILTER))
- (authorized-log-user? user (get-topo-port-workerlog %) *STORM-CONF*)) logs))]
- (sort #(compare (.lastModified %2) (.lastModified %1))
- (filter-authorized-fn
- user
- (filter #(re-find worker-log-filename-pattern (.getName %)) (DirectoryCleaner/getFilesForDir port-dir))))))
-
-(defn deep-search-logs-for-topology
- [topology-id user ^String root-dir search num-matches port file-offset offset search-archived? callback origin]
- (json-response
- (if (or (not search) (not (.exists (File. (str root-dir ServerUtils/FILE_PATH_SEPARATOR topology-id)))))
- []
- (let [file-offset (if file-offset (Integer/parseInt file-offset) 0)
- offset (if offset (Integer/parseInt offset) 0)
- num-matches (or (Integer/parseInt num-matches) 1)
- port-dirs (vec (.listFiles (File. (str root-dir ServerUtils/FILE_PATH_SEPARATOR topology-id))))
- logs-for-port-fn (partial logs-for-port user)]
- (if (or (not port) (= "*" port))
- ;; Check for all ports
- (let [filtered-logs (filter (comp not empty?) (map logs-for-port-fn port-dirs))]
- (if search-archived?
- (map #(find-n-matches % num-matches 0 0 search)
- filtered-logs)
- (map #(find-n-matches % num-matches 0 0 search)
- (map (comp vector first) filtered-logs))))
- ;; Check just the one port
- (if (not (contains? (into #{} (map str (*STORM-CONF* SUPERVISOR-SLOTS-PORTS))) port))
- []
- (let [port-dir (File. (str root-dir ServerUtils/FILE_PATH_SEPARATOR topology-id ServerUtils/FILE_PATH_SEPARATOR port))]
- (if (or (not (.exists port-dir)) (empty? (logs-for-port user port-dir)))
- []
- (let [filtered-logs (logs-for-port user port-dir)]
- (if search-archived?
- (find-n-matches filtered-logs num-matches file-offset offset search)
- (find-n-matches [(first filtered-logs)] num-matches 0 offset search)))))))))
- callback
- :headers {"Access-Control-Allow-Origin" origin
- "Access-Control-Allow-Credentials" "true"}))
-
-(defn log-template
- ([body] (log-template body nil nil))
- ([body fname user]
- (html4
- [:head
- [:title (str (escape-html fname) " - Storm Log Viewer")]
- (include-css "/css/bootstrap-3.3.1.min.css")
- (include-css "/css/jquery.dataTables.1.10.4.min.css")
- (include-css "/css/style.css")
- ]
- [:body
- (concat
- (when (not (blank? user)) [[:div.ui-user [:p "User: " user]]])
- [[:div.ui-note [:p "Note: the drop-list shows at most 1024 files for each worker directory."]]]
- [[:h3 (escape-html fname)]]
- (seq body))
- ])))
-
-(def http-creds-handler (AuthUtils/GetUiHttpCredentialsPlugin *STORM-CONF*))
-
-(defn- parse-long-from-map [m k]
- (try
- (Long/parseLong (k m))
- (catch NumberFormatException ex
- (throw (InvalidRequestException.
- (str "Could not make an integer out of the query parameter '"
- (name k) "'")
- ex)))))
-
-(defn list-log-files
- [user topoId port log-root callback origin]
- (let [file-results
- (if (nil? topoId)
- (if (nil? port)
- (get-all-logs-for-rootdir (File. log-root))
- (reduce concat
- (for [topo-dir (.listFiles (File. log-root))]
- (reduce concat
- (for [port-dir (.listFiles topo-dir)]
- (if (= (str port) (.getName port-dir))
- (into [] (DirectoryCleaner/getFilesForDir port-dir))))))))
- (if (nil? port)
- (let [topo-dir (File. (str log-root ServerUtils/FILE_PATH_SEPARATOR topoId))]
- (if (.exists topo-dir)
- (reduce concat
- (for [port-dir (.listFiles topo-dir)]
- (into [] (DirectoryCleaner/getFilesForDir port-dir))))
- []))
- (let [port-dir (ConfigUtils/getWorkerDirFromRoot log-root topoId port)]
- (if (.exists port-dir)
- (into [] (DirectoryCleaner/getFilesForDir port-dir))
- []))))
- file-strs (sort (for [file file-results]
- (get-topo-port-workerlog file)))]
- (json-response file-strs
- callback
- :headers {"Access-Control-Allow-Origin" origin
- "Access-Control-Allow-Credentials" "true"})))
-
-(defn get-profiler-dump-files
- [dir]
- (filter (comp not nil?)
- (for [f (DirectoryCleaner/getFilesForDir dir)]
- (let [name (.getName f)]
- (if (or
- (.endsWith name ".txt")
- (.endsWith name ".jfr")
- (.endsWith name ".bin"))
- (.getName f))))))
-
-(defroutes log-routes
- (GET "/log" [:as req & m]
- (try
- (.mark logviewer:num-log-page-http-requests)
- (let [servlet-request (:servlet-request req)
- log-root (:log-root req)
- user (.getUserName http-creds-handler servlet-request)
- start (if (:start m) (parse-long-from-map m :start))
- length (if (:length m) (parse-long-from-map m :length))
- file (URLDecoder/decode (:file m))]
- (set-log-file-permissions file log-root)
- (log-template (log-page file start length (:grep m) user log-root)
- file user))
- (catch InvalidRequestException ex
- (log-error ex)
- (ring-response-from-exception ex))))
- (GET "/dumps/:topo-id/:host-port/:filename"
- [:as {:keys [servlet-request servlet-response log-root]} topo-id host-port filename &m]
- (let [user (.getUserName http-creds-handler servlet-request)
- port (second (split host-port #":"))
- dir (File. (str log-root
- ServerUtils/FILE_PATH_SEPARATOR
- topo-id
- ServerUtils/FILE_PATH_SEPARATOR
- port))
- file (File. (str log-root
- ServerUtils/FILE_PATH_SEPARATOR
- topo-id
- ServerUtils/FILE_PATH_SEPARATOR
- port
- ServerUtils/FILE_PATH_SEPARATOR
- filename))]
- (if (and (.exists dir) (.exists file))
- (if (or (blank? (*STORM-CONF* UI-FILTER))
- (authorized-log-user? user
- (str topo-id ServerUtils/FILE_PATH_SEPARATOR port ServerUtils/FILE_PATH_SEPARATOR "worker.log")
- *STORM-CONF*))
- (-> (resp/response file)
- (resp/content-type "application/octet-stream"))
- (unauthorized-user-html user))
- (-> (resp/response "Page not found")
- (resp/status 404)))))
- (GET "/dumps/:topo-id/:host-port"
- [:as {:keys [servlet-request servlet-response log-root]} topo-id host-port &m]
- (let [user (.getUserName http-creds-handler servlet-request)
- port (second (split host-port #":"))
- dir (File. (str log-root
- ServerUtils/FILE_PATH_SEPARATOR
- topo-id
- ServerUtils/FILE_PATH_SEPARATOR
- port))]
- (if (.exists dir)
- (if (or (blank? (*STORM-CONF* UI-FILTER))
- (authorized-log-user? user
- (str topo-id ServerUtils/FILE_PATH_SEPARATOR port ServerUtils/FILE_PATH_SEPARATOR "worker.log")
- *STORM-CONF*))
- (html4
- [:head
- [:title "File Dumps - Storm Log Viewer"]
- (include-css "/css/bootstrap-3.3.1.min.css")
- (include-css "/css/jquery.dataTables.1.10.4.min.css")
- (include-css "/css/style.css")]
- [:body
- [:ul
- (for [file (get-profiler-dump-files dir)]
- [:li
- [:a {:href (str "/dumps/" topo-id "/" host-port "/" file)} file ]])]])
- (unauthorized-user-html user))
- (-> (resp/response "Page not found")
- (resp/status 404)))))
- (GET "/daemonlog" [:as req & m]
- (try
- (.mark logviewer:num-daemonlog-page-http-requests)
- (let [servlet-request (:servlet-request req)
- daemonlog-root (:daemonlog-root req)
- user (.getUserName http-creds-handler servlet-request)
- start (if (:start m) (parse-long-from-map m :start))
- length (if (:length m) (parse-long-from-map m :length))
- file (URLDecoder/decode (:file m))]
- (log-template (daemonlog-page file start length (:grep m) user daemonlog-root)
- file user))
- (catch InvalidRequestException ex
- (log-error ex)
- (ring-response-from-exception ex))))
- (GET "/download" [:as {:keys [servlet-request servlet-response log-root]} & m]
- (try
- (.mark logviewer:num-download-log-file-http-requests)
- (let [user (.getUserName http-creds-handler servlet-request)
- file (URLDecoder/decode (:file m))]
- (set-log-file-permissions file log-root)
- (download-log-file file servlet-request servlet-response user log-root))
- (catch InvalidRequestException ex
- (log-error ex)
- (ring-response-from-exception ex))))
- (GET "/daemondownload/:file" [:as {:keys [servlet-request servlet-response daemonlog-root]} file & m]
- (try
- (.mark logviewer:num-download-log-daemon-file-http-requests)
- (let [user (.getUserName http-creds-handler servlet-request)]
- (download-log-file file servlet-request servlet-response user daemonlog-root :is-daemon true))
- (catch InvalidRequestException ex
- (log-error ex)
- (ring-response-from-exception ex))))
- (GET "/search/:file" [:as {:keys [servlet-request servlet-response log-root daemonlog-root]} file & m]
- ;; We do not use servlet-response here, but do not remove it from the
- ;; :keys list, or this rule could stop working when an authentication
- ;; filter is configured.
- (try
- (let [user (.getUserName http-creds-handler servlet-request)
- is-daemon (= (:is-daemon m) "yes")]
- (search-log-file (URLDecoder/decode file)
- user
- (if is-daemon daemonlog-root log-root)
- is-daemon
- (:search-string m)
- (:num-matches m)
- (:start-byte-offset m)
- (:callback m)
- (.getHeader servlet-request "Origin")))
- (catch InvalidRequestException ex
- (log-error ex)
- (json-response (UIHelpers/exceptionToJson ex) (:callback m) :status 400))))
- (GET "/deepSearch/:topo-id" [:as {:keys [servlet-request servlet-response log-root]} topo-id & m]
- ;; We do not use servlet-response here, but do not remove it from the
- ;; :keys list, or this rule could stop working when an authentication
- ;; filter is configured.
- (try
- (let [user (.getUserName http-creds-handler servlet-request)]
- (deep-search-logs-for-topology topo-id
- user
- log-root
- (:search-string m)
- (:num-matches m)
- (:port m)
- (:start-file-offset m)
- (:start-byte-offset m)
- (:search-archived m)
- (:callback m)
- (.getHeader servlet-request "Origin")))
- (catch InvalidRequestException ex
- (log-error ex)
- (json-response (UIHelpers/exceptionToJson ex) (:callback m) :status 400))))
- (GET "/searchLogs" [:as req & m]
- (try
- (let [servlet-request (:servlet-request req)
- user (.getUserName http-creds-handler servlet-request)]
- (list-log-files user
- (:topoId m)
- (:port m)
- (:log-root req)
- (:callback m)
- (.getHeader servlet-request "Origin")))
- (catch InvalidRequestException ex
- (log-error ex)
- (json-response (UIHelpers/exceptionToJson ex) (:callback m) :status 400))))
- (GET "/listLogs" [:as req & m]
- (try
- (.mark logviewer:num-list-logs-http-requests)
- (let [servlet-request (:servlet-request req)
- user (.getUserName http-creds-handler servlet-request)]
- (list-log-files user
- (:topoId m)
- (:port m)
- (:log-root req)
- (:callback m)
- (.getHeader servlet-request "Origin")))
- (catch InvalidRequestException ex
- (log-error ex)
- (json-response (UIHelpers/exceptionToJson ex) (:callback m) :status 400))))
- (route/resources "/")
- (route/not-found "Page not found"))
-
-(defn conf-middleware
- "For passing the storm configuration with each request."
- [app log-root daemonlog-root]
- (fn [req]
- (app (assoc req :log-root log-root :daemonlog-root daemonlog-root))))
-
-(defn start-logviewer! [conf log-root-dir daemonlog-root-dir]
- (try
- (let [header-buffer-size (int (.get conf UI-HEADER-BUFFER-BYTES))
- filter-class (conf UI-FILTER)
- filter-params (conf UI-FILTER-PARAMS)
- logapp (handler/api (-> log-routes
- requests-middleware)) ;; query params as map
- middle (conf-middleware logapp log-root-dir daemonlog-root-dir)
- filters-confs (if (conf UI-FILTER)
- [(FilterConfiguration. filter-class (or (conf UI-FILTER-PARAMS) {}))]
- [])
- filters-confs (concat filters-confs
- [(FilterConfiguration. "org.eclipse.jetty.servlets.GzipFilter" "Gzipper" {})])
- https-port (int (or (conf LOGVIEWER-HTTPS-PORT) 0))
- keystore-path (conf LOGVIEWER-HTTPS-KEYSTORE-PATH)
- keystore-pass (conf LOGVIEWER-HTTPS-KEYSTORE-PASSWORD)
- keystore-type (conf LOGVIEWER-HTTPS-KEYSTORE-TYPE)
- key-password (conf LOGVIEWER-HTTPS-KEY-PASSWORD)
- truststore-path (conf LOGVIEWER-HTTPS-TRUSTSTORE-PATH)
- truststore-password (conf LOGVIEWER-HTTPS-TRUSTSTORE-PASSWORD)
- truststore-type (conf LOGVIEWER-HTTPS-TRUSTSTORE-TYPE)
- want-client-auth (conf LOGVIEWER-HTTPS-WANT-CLIENT-AUTH)
- need-client-auth (conf LOGVIEWER-HTTPS-NEED-CLIENT-AUTH)]
- (UIHelpers/stormRunJetty (int (conf LOGVIEWER-PORT))
- (reify IConfigurator (execute [this server]
- (UIHelpers/configSsl server
- https-port
- keystore-path
- keystore-pass
- keystore-type
- key-password
- truststore-path
- truststore-password
- truststore-type
- want-client-auth
- need-client-auth)
- (UIHelpers/configFilter server (ring.util.servlet/servlet middle) filters-confs)))))
- (catch Exception ex
- (log-error ex))))
-
-(defn -main []
- (let [conf (clojurify-structure (ConfigUtils/readStormConfig))
- log-root (ConfigUtils/workerArtifactsRoot conf)
- daemonlog-root (log-root-dir (conf LOGVIEWER-APPENDER-NAME))]
- (Utils/setupDefaultUncaughtExceptionHandler)
- (start-log-cleaner! conf log-root)
- (log-message "Starting logviewer server for storm version '"
- STORM-VERSION
- "'")
- (start-logviewer! conf log-root daemonlog-root)
- (StormMetricsRegistry/startMetricsReporters conf)))
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/src/dev/logviewer-search-context-tests.log.gz
----------------------------------------------------------------------
diff --git a/storm-core/src/dev/logviewer-search-context-tests.log.gz b/storm-core/src/dev/logviewer-search-context-tests.log.gz
deleted file mode 100644
index 5cf2a06..0000000
Binary files a/storm-core/src/dev/logviewer-search-context-tests.log.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/src/dev/logviewer-search-context-tests.log.test
----------------------------------------------------------------------
diff --git a/storm-core/src/dev/logviewer-search-context-tests.log.test b/storm-core/src/dev/logviewer-search-context-tests.log.test
deleted file mode 100644
index 6e4d4af..0000000
--- a/storm-core/src/dev/logviewer-search-context-tests.log.test
+++ /dev/null
@@ -1 +0,0 @@
-needle needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle needle
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/src/dev/small-worker.log.test
----------------------------------------------------------------------
diff --git a/storm-core/src/dev/small-worker.log.test b/storm-core/src/dev/small-worker.log.test
deleted file mode 100644
index 27d61d1..0000000
--- a/storm-core/src/dev/small-worker.log.test
+++ /dev/null
@@ -1 +0,0 @@
-000000 needle 000000
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/src/dev/test-3072.log.test
----------------------------------------------------------------------
diff --git a/storm-core/src/dev/test-3072.log.test b/storm-core/src/dev/test-3072.log.test
deleted file mode 100644
index 56dc6f1..0000000
--- a/storm-core/src/dev/test-3072.log.test
+++ /dev/null
@@ -1,3 +0,0 @@
-This is a test log file of size 3072.
-
-.....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
.....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
.....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
....................................needle
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/src/dev/test-worker.log.test
----------------------------------------------------------------------
diff --git a/storm-core/src/dev/test-worker.log.test b/storm-core/src/dev/test-worker.log.test
deleted file mode 100644
index 8fb4c53..0000000
--- a/storm-core/src/dev/test-worker.log.test
+++ /dev/null
@@ -1,380 +0,0 @@
-Test needle is near the beginning of the file.
-This file assumes a buffer size of 2048 bytes, a max search string size of 1024 bytes, and a context length of 128 UTF-8 characters.
-The early match tests the case when we find a match too close to the start of the file to give the normal before context strings.
-
-padding 5
-padding 6
-padding 7
-padding 8
-padding 9
-padding 10
-padding 11
-padding 12
-padding 13
-padding 14
-padding 15
-padding 16
-padding 17
-padding 18
-padding 19
-padding 20
-padding 21
-padding 22
-padding 23
-padding 24
-padding 25
-padding 26
-padding 27
-padding 28
-padding 29
-padding 30
-padding 31
-padding 32
-padding 33
-padding 34
-padding 35
-padding 36
-padding 37
-padding 38
-padding 39
-padding 40
-padding 41
-padding 42
-padding 43
-padding 44
-padding 45
-padding 46
-padding 47
-padding 48
-padding 49
-padding 50
-padding 51
-padding 52
-padding 53
-padding 54
-padding 55
-padding 56
-padding 57
-padding 58
-padding 59
-padding 60
-padding 61
-padding 62
-padding 63
-padding 64
-padding 65
-padding 66
-padding 67
-padding 68
-padding 69
-padding 70
-padding 71
-padding 72
-padding 73
-padding 74
-padding 75
-padding 76
-padding 77
-padding 78
-padding 79
-padding 80
-padding 81
-padding 82
-padding 83
-padding 84
-padding 85
-padding 86
-padding 87
-padding 88
-padding 89
-padding 90
-padding 91
-padding 92
-padding 93
-padding 94
-padding 95
-padding 96
-padding 97
-padding 98
-padding 99
-padding 100
-padding 101
-padding 102
-padding 103
-padding 104
-padding 105
-padding 106
-padding 107
-padding 108
-padding 109
-padding 110
-padding 111
-padding 112
-padding 113
-padding 114
-padding 115
-padding 116
-padding 117
-padding 118
-padding 119
-padding 120
-padding 121
-padding 122
-padding 123
-padding 124
-padding 125
-padding 126
-padding 127
-padding 128
-padding 129
-padding 130
-padding 131
-padding 132
-padding 133
-padding 134
-padding 135
-padding 136
-padding 137
-padding 138
-padding 139
-padding 140
-padding 141
-padding 142
-padding 143
-padding 144
-padding 145
-padding 146
-padding 147
-padding 148
-padding 149
-padding 150
-padding 151
-padding 152
-padding 153
-Near the end of a 1024 byte block, a needle.
-A needle that straddles a 1024 byte boundary should also be detected.
-
-padding 157
-padding 158
-padding 159
-padding 160
-padding 161
-padding 162
-padding 163
-padding 164
-padding 165
-padding 166
-padding 167
-padding 168
-padding 169
-padding 170
-padding 171
-padding 172
-padding 173
-padding 174
-padding 175
-padding 176
-padding 177
-padding 178
-padding 179
-padding 180
-padding 181
-padding 182
-padding 183
-padding 184
-padding 185
-padding 186
-padding 187
-padding 188
-padding 189
-padding 190
-padding 191
-padding 192
-padding 193
-padding 194
-padding 195
-padding 196
-padding 197
-padding 198
-padding 199
-padding 200
-padding 201
-padding 202
-padding 203
-padding 204
-padding 205
-padding 206
-padding 207
-padding 208
-padding 209
-padding 210
-padding 211
-padding 212
-padding 213
-padding 214
-padding 215
-padding 216
-padding 217
-padding 218
-padding 219
-padding 220
-padding 221
-padding 222
-padding 223
-padding 224
-padding 225
-padding 226
-padding 227
-padding 228
-padding 229
-padding 230
-padding 231
-padding 232
-padding 233
-padding 234
-padding 235
-
-
-Here a needle occurs just after a 1024 byte boundary. It should have the correct context.
-
-Text with two adjoining matches: needleneedle
-
-padding 243
-padding 244
-padding 245
-padding 246
-padding 247
-padding 248
-padding 249
-padding 250
-padding 251
-padding 252
-padding 253
-padding 254
-padding 255
-padding 256
-padding 257
-padding 258
-padding 259
-padding 260
-padding 261
-padding 262
-padding 263
-padding 264
-padding 265
-padding 266
-padding 267
-padding 268
-padding 269
-padding 270
-padding 271
-padding 272
-padding 273
-padding 274
-padding 275
-padding 276
-padding 277
-padding 278
-padding 279
-padding 280
-padding 281
-padding 282
-padding 283
-padding 284
-padding 285
-padding 286
-padding 287
-padding 288
-padding 289
-padding 290
-padding 291
-padding 292
-padding 293
-padding 294
-padding 295
-padding 296
-padding 297
-padding 298
-padding 299
-padding 300
-padding 301
-padding 302
-padding 303
-padding 304
-
-The following match of 1024 bytes completely fills half the byte buffer. It is a search substring of the maximum size......
-
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
XXXXXXXXXXXXXXXXXXXXXXXXXXX
-The following max-size match straddles a 1024 byte buffer.
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
XXXXXXXXXXXXXXXXXXXXXXXXXXX
-
-Here are four non-ascii 1-byte UTF-8 characters: αβγδε
-
-needle
-
-Here are four printable 2-byte UTF-8 characters: ¡¢£¤¥
-
-needle
-
-
-
-Here are four printable 3-byte UTF-8 characters: ऄअआइई
-
-needle
-
-Here are four printable 4-byte UTF-8 characters: 𐄇𐄈𐄉𐄋𐄌
-
-needle
-
-Here are four of the same invalid UTF-8 characters: ����������������
-
-needle
-
-padding 332
-padding 333
-padding 334
-padding 335
-padding 336
-padding 337
-padding 338
-padding 339
-padding 340
-padding 341
-padding 342
-padding 343
-padding 344
-padding 345
-padding 346
-padding 347
-padding 348
-padding 349
-padding 350
-padding 351
-padding 352
-padding 353
-padding 354
-padding 355
-padding 356
-padding 357
-padding 358
-padding 359
-padding 360
-padding 361
-padding 362
-padding 363
-padding 364
-padding 365
-padding 366
-padding 367
-padding 368
-padding 369
-padding 370
-padding 371
-padding 372
-padding 373
-padding 374
-padding 375
-
-The following tests multibyte UTF-8 Characters straddling the byte boundary: 𐄀𐄁𐄂
-
-needle
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
deleted file mode 100644
index dc76157..0000000
--- a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.daemon;
-
-import java.io.IOException;
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.DirectoryStream;
-import java.util.Stack;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.Comparator;
-import java.util.PriorityQueue;
-import java.util.regex.Pattern;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Provide methods to help Logviewer to clean up
- * files in directories and to get a list of files without
- * worrying about excessive memory usage.
- *
- */
-public class DirectoryCleaner {
- private static final Logger LOG = LoggerFactory.getLogger(DirectoryCleaner.class);
- // used to recognize the pattern of active log files, we may remove the "current" from this list
- private static final Pattern ACTIVE_LOG_PATTERN = Pattern.compile(".*\\.(log|err|out|current|yaml|pid)$");
- // used to recognize the pattern of some meta files in a worker log directory
- private static final Pattern META_LOG_PATTERN= Pattern.compile(".*\\.(yaml|pid)$");
-
- // not defining this as static is to allow for mocking in tests
- public DirectoryStream<Path> getStreamForDirectory(File dir) throws IOException {
- DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath());
- return stream;
- }
-
- /**
- * If totalSize of files exceeds the either the per-worker quota or global quota,
- * Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs.
- * We use the parameter forPerDir to switch between the two deletion modes.
- * @param dirs the list of directories to be scanned for deletion
- * @param quota the per-dir quota or the total quota for the all directories
- * @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories globally
- * @param activeDirs only for global deletion, we want to skip the active logs in activeDirs
- * @return number of files deleted
- */
- public int deleteOldestWhileTooLarge(List<File> dirs,
- long quota, boolean forPerDir, Set<String> activeDirs) throws IOException {
- final int PQ_SIZE = 1024; // max number of files to delete for every round
- final int MAX_ROUNDS = 512; // max rounds of scanning the dirs
- long totalSize = 0;
- int deletedFiles = 0;
-
- for (File dir : dirs) {
- try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
- for (Path path : stream) {
- File file = path.toFile();
-
- if (isFileEligibleToSkipDelete(forPerDir, activeDirs, dir, file)) {
- continue; // skip adding length
- }
-
- totalSize += file.length();
- }
- }
- }
-
- LOG.debug("totalSize: {} quota: {}", totalSize, quota);
- long toDeleteSize = totalSize - quota;
- if (toDeleteSize <= 0) {
- return deletedFiles;
- }
-
- Comparator<File> comparator = new Comparator<File>() {
- public int compare(File f1, File f2) {
- if (f1.lastModified() > f2.lastModified()) {
- return -1;
- } else {
- return 1;
- }
- }
- };
- // the oldest pq_size files in this directory will be placed in PQ, with the newest at the root
- PriorityQueue<File> pq = new PriorityQueue<File>(PQ_SIZE, comparator);
- int round = 0;
- while (toDeleteSize > 0) {
- LOG.debug("To delete size is {}, start a new round of deletion, round: {}", toDeleteSize, round);
- for (File dir : dirs) {
- try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
- for (Path path : stream) {
- File file = path.toFile();
- if (isFileEligibleToSkipDelete(forPerDir, activeDirs, dir, file)) {
- continue;
- }
- if (pq.size() < PQ_SIZE) {
- pq.offer(file);
- } else {
- if (file.lastModified() < pq.peek().lastModified()) {
- pq.poll();
- pq.offer(file);
- }
- }
- }
- }
- }
- // need to reverse the order of elements in PQ to delete files from oldest to newest
- Stack<File> stack = new Stack<File>();
- while (!pq.isEmpty()) {
- File file = pq.poll();
- stack.push(file);
- }
- while (!stack.isEmpty() && toDeleteSize > 0) {
- File file = stack.pop();
- toDeleteSize -= file.length();
- LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getCanonicalPath(), file.length(), file.lastModified());
- file.delete();
- deletedFiles++;
- }
- pq.clear();
- round++;
- if (round >= MAX_ROUNDS) {
- if (forPerDir) {
- LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " +
- "a single directory : {}, will delete the rest files in next interval.",
- MAX_ROUNDS, dirs.get(0).getCanonicalPath());
- } else {
- LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " +
- "will delete the rest files in next interval.", MAX_ROUNDS);
- }
- break;
- }
- }
- return deletedFiles;
- }
-
- private boolean isFileEligibleToSkipDelete(boolean forPerDir, Set<String> activeDirs, File dir, File file) throws IOException {
- if (forPerDir) {
- if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
- return true;
- }
- } else { // for global cleanup
- if (activeDirs.contains(dir.getCanonicalPath())) { // for an active worker's dir, make sure for the last "/"
- if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
- return true;
- }
- } else {
- if (META_LOG_PATTERN.matcher(file.getName()).matches()) {
- return true;
- }
- }
- }
- return false;
- }
-
- // Note that to avoid memory problem, we only return the first 1024 files in a directory
- public static List<File> getFilesForDir(File dir) throws IOException {
- List<File> files = new ArrayList<File>();
- final int MAX_NUM = 1024;
-
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath())) {
- for (Path path : stream) {
- files.add(path.toFile());
- if (files.size() >= MAX_NUM) {
- break;
- }
- }
- }
- return files;
- }
-}
[15/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* apply changed API path to integration-test
* didn't realize integration accesses logviewer
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/37f9c599
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/37f9c599
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/37f9c599
Branch: refs/heads/master
Commit: 37f9c59949a1d8873668934b317e5c1ba20d3599
Parents: f9ed5d4
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Jul 20 19:51:17 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Jul 20 19:51:17 2017 +0900
----------------------------------------------------------------------
.../src/test/java/org/apache/storm/st/wrapper/TopoWrap.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/37f9c599/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
index af1e1be..a0082a8 100644
--- a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
+++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
@@ -261,8 +261,8 @@ public class TopoWrap {
public ExecutorURL(String componentId, String host, int logViewerPort, int executorPort, String topoId) throws MalformedURLException {
String sep = "%2F"; //hex of "/"
- String viewUrlStr = String.format("http://%s:%s/log?file=", host, logViewerPort);
- String downloadUrlStr = String.format("http://%s:%s/download?file=%%2F", host, logViewerPort);
+ String viewUrlStr = String.format("http://%s:%s/api/v1/log?file=", host, logViewerPort);
+ String downloadUrlStr = String.format("http://%s:%s/api/v1/download?file=%%2F", host, logViewerPort);
viewUrl = new URL(String.format("%s/%s%s%d%sworker.log", viewUrlStr, topoId, sep, executorPort, sep));
downloadUrl = new URL(String.format("%s/%s%s%d%sworker.log", downloadUrlStr, topoId, sep, executorPort, sep));
this.componentId = componentId;
[09/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java
new file mode 100644
index 0000000..c0a69cb
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogSearchHandler.java
@@ -0,0 +1,686 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.handler;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.DirectoryCleaner;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.URLBuilder;
+import org.apache.storm.daemon.wip.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.wip.logviewer.utils.WorkerLogs;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.json.simple.JSONAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.drop;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.first;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.last;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.rest;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+import static org.apache.storm.daemon.wip.logviewer.LogviewerConstant.DEFAULT_BYTES_PER_PAGE;
+
+public class LogviewerLogSearchHandler {
+ private final static Logger LOG = LoggerFactory.getLogger(LogviewerLogSearchHandler.class);
+
+ public static final int GREP_MAX_SEARCH_SIZE = 1024;
+ public static final int GREP_BUF_SIZE = 2048;
+ public static final int GREP_CONTEXT_SIZE = 128;
+ public static final Pattern WORKER_LOG_FILENAME_PATTERN = Pattern.compile("^worker.log(.*)");
+
+ private final Map<String, Object> stormConf;
+ private final String logRoot;
+ private final String daemonLogRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+ private final Integer logviewerPort;
+
+ public LogviewerLogSearchHandler(Map<String, Object> stormConf, String logRoot, String daemonLogRoot,
+ ResourceAuthorizer resourceAuthorizer) {
+ this.stormConf = stormConf;
+ this.logRoot = logRoot;
+ this.daemonLogRoot = daemonLogRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+
+ this.logviewerPort = ObjectReader.getInt(stormConf.get(DaemonConfig.LOGVIEWER_PORT));
+ }
+
+ public Response searchLogFile(String fileName, String user, boolean isDaemon, String search,
+ String numMatchesStr, String offsetStr, String callback, String origin)
+ throws IOException, InvalidRequestException {
+ String rootDir = isDaemon ? daemonLogRoot : logRoot;
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ Response response;
+ if (file.exists()) {
+ if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
+ Integer numMatchesInt = numMatchesStr != null ? tryParseIntParam("num-matches", numMatchesStr) : null;
+ Integer offsetInt = offsetStr != null ? tryParseIntParam("start-byte-offset", offsetStr) : null;
+
+ try {
+ if (StringUtils.isNotEmpty(search) && search.getBytes("UTF-8").length <= GREP_MAX_SEARCH_SIZE) {
+ Map<String, Object> entity = new HashMap<>();
+ entity.put("isDaemon", isDaemon ? "yes" : "no");
+ entity.putAll(substringSearch(file, search, isDaemon, numMatchesInt, offsetInt));
+
+ response = LogviewerResponseBuilder.buildSuccessJsonResponse(entity, callback, origin);
+ } else {
+ throw new InvalidRequestException("Search substring must be between 1 and 1024 "
+ + "UTF-8 bytes in size (inclusive)");
+ }
+ } catch (Exception ex) {
+ response = LogviewerResponseBuilder.buildExceptionJsonResponse(ex, callback);
+ }
+ } else {
+ // unauthorized
+ response = LogviewerResponseBuilder.buildUnauthorizedUserJsonResponse(user, callback);
+ }
+ } else {
+ // not found
+ Map<String, String> entity = new HashMap<>();
+ entity.put("error", "Not Found");
+ entity.put("errorMessage", "The file was not found on this node.");
+
+ response = new JsonResponseBuilder().setData(entity).setCallback(callback).setStatus(404).build();
+ }
+
+ return response;
+ }
+
+ public Response deepSearchLogsForTopology(String topologyId, String user, String search,
+ String numMatchesStr, String portStr, String fileOffsetStr, String offsetStr,
+ Boolean searchArchived, String callback, String origin) {
+ String rootDir = logRoot;
+ Object returnValue;
+ File topologyDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId);
+ if (StringUtils.isEmpty(search) || !topologyDir.exists()) {
+ returnValue = new ArrayList<>();
+ } else {
+ int fileOffset = ObjectReader.getInt(fileOffsetStr, 0);
+ int offset = ObjectReader.getInt(offsetStr, 0);
+ int numMatches = ObjectReader.getInt(numMatchesStr, 1);
+
+ File[] portDirsArray = topologyDir.listFiles();
+ List<File> portDirs;
+ if (portDirsArray != null) {
+ portDirs = Arrays.asList(portDirsArray);
+ } else {
+ portDirs = new ArrayList<>();
+ }
+
+ if (StringUtils.isEmpty(portStr) || portStr.equals("*")) {
+ // check for all ports
+ List<List<File>> filteredLogs = portDirs.stream()
+ .map(portDir -> logsForPort(user, portDir))
+ .filter(logs -> logs != null && !logs.isEmpty())
+ .collect(toList());
+
+ if (BooleanUtils.isTrue(searchArchived)) {
+ returnValue = filteredLogs.stream()
+ .map(fl -> findNMatches(fl, numMatches, 0, 0, search))
+ .collect(toList());
+ } else {
+ returnValue = filteredLogs.stream()
+ .map(fl -> Collections.singletonList(first(fl)))
+ .map(fl -> findNMatches(fl, numMatches, 0, 0, search))
+ .collect(toList());
+ }
+ } else {
+ int port = Integer.parseInt(portStr);
+ // check just the one port
+ List<Integer> slotsPorts = (List<Integer>) stormConf.getOrDefault(DaemonConfig.SUPERVISOR_SLOTS_PORTS,
+ new ArrayList<>());
+ boolean containsPort = slotsPorts.stream()
+ .anyMatch(slotPort -> slotPort != null && (slotPort == port));
+ if (!containsPort) {
+ returnValue = new ArrayList<>();
+ } else {
+ File portDir = new File(rootDir + Utils.FILE_PATH_SEPARATOR + topologyId +
+ Utils.FILE_PATH_SEPARATOR + port);
+
+ if (!portDir.exists() || logsForPort(user, portDir).isEmpty()) {
+ returnValue = new ArrayList<>();
+ } else {
+ List<File> filteredLogs = logsForPort(user, portDir);
+ if (BooleanUtils.isTrue(searchArchived)) {
+ returnValue = findNMatches(filteredLogs, numMatches, fileOffset, offset, search);
+ } else {
+ returnValue = findNMatches(Collections.singletonList(first(filteredLogs)),
+ numMatches, 0, offset, search);
+ }
+ }
+ }
+ }
+ }
+
+ return LogviewerResponseBuilder.buildSuccessJsonResponse(returnValue, callback, origin);
+ }
+
+ private Integer tryParseIntParam(String paramName, String value) throws InvalidRequestException {
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new InvalidRequestException("Could not parse " + paramName + " to an integer");
+ }
+ }
+
+ private Map<String,Object> substringSearch(File file, String searchString, int numMatches, int startByteOffset) throws InvalidRequestException {
+ return substringSearch(file, searchString, false, numMatches, startByteOffset);
+ }
+
+ private Map<String,Object> substringSearch(File file, String searchString, boolean isDaemon, Integer numMatches, Integer startByteOffset) throws InvalidRequestException {
+ try {
+ if (StringUtils.isEmpty(searchString)) {
+ throw new IllegalArgumentException("Precondition fails: search string should not be empty.");
+ }
+ if (searchString.getBytes("UTF-8").length > GREP_MAX_SEARCH_SIZE) {
+ throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + GREP_MAX_SEARCH_SIZE);
+ }
+
+ boolean isZipFile = file.getName().endsWith(".gz");
+ FileInputStream fis = new FileInputStream(file);
+ InputStream gzippedInputStream;
+ if (isZipFile) {
+ gzippedInputStream = new GZIPInputStream(fis);
+ } else {
+ gzippedInputStream = fis;
+ }
+
+ BufferedInputStream stream = new BufferedInputStream(gzippedInputStream);
+
+ int fileLength;
+ if (isZipFile) {
+ fileLength = (int) ServerUtils.zipFileSize(file);
+ } else {
+ fileLength = (int) file.length();
+ }
+
+ ByteBuffer buf = ByteBuffer.allocate(GREP_BUF_SIZE);
+ byte[] bufArray = buf.array();
+ int totalBytesRead = 0;
+ byte[] searchBytes = searchString.getBytes("UTF-8");
+ numMatches = numMatches != null ? numMatches : 10;
+ startByteOffset = startByteOffset != null ? startByteOffset : 0;
+
+ // Start at the part of the log file we are interested in.
+ // Allow searching when start-byte-offset == file-len so it doesn't blow up on 0-length files
+ if (startByteOffset > fileLength) {
+ throw new InvalidRequestException("Cannot search past the end of the file");
+ }
+
+ if (startByteOffset > 0) {
+ StreamUtil.skipBytes(stream, startByteOffset);
+ }
+
+ Arrays.fill(bufArray, (byte) 0);
+
+ int bytesRead = stream.read(bufArray, 0, Math.min((int) fileLength, GREP_BUF_SIZE));
+ buf.limit(bytesRead);
+ totalBytesRead += bytesRead;
+
+ List<Map<String, Object>> initialMatches = new ArrayList<>();
+ int initBufOffset = 0;
+ int byteOffset = startByteOffset;
+ byte[] beforeBytes = null;
+
+ Map<String, Object> ret = new HashMap<>();
+ while (true) {
+ SubstringSearchResult searchRet = bufferSubstringSearch(isDaemon, file, fileLength, byteOffset, initBufOffset,
+ stream, startByteOffset, totalBytesRead, buf, searchBytes, initialMatches, numMatches, beforeBytes);
+
+ List<Map<String, Object>> matches = searchRet.getMatches();
+ Integer newByteOffset = searchRet.getNewByteOffset();
+ byte[] newBeforeBytes = searchRet.getNewBeforeBytes();
+
+ if (matches.size() < numMatches && totalBytesRead + startByteOffset < fileLength) {
+ // The start index is positioned to find any possible
+ // occurrence search string that did not quite fit in the
+ // buffer on the previous read.
+ int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length;
+
+ totalBytesRead = rotateGrepBuffer(buf, stream, totalBytesRead, file, fileLength);
+ if (totalBytesRead < 0) {
+ throw new InvalidRequestException("Cannot search past the end of the file");
+ }
+
+ initialMatches = matches;
+ initBufOffset = newBufOffset;
+ byteOffset = newByteOffset;
+ beforeBytes = newBeforeBytes;
+ } else {
+ ret.put("isDaemon", isDaemon ? "yes" : "no");
+ Integer nextByteOffset = null;
+ if (matches.size() >= numMatches || totalBytesRead < fileLength) {
+ nextByteOffset = (Integer) last(matches).get("byteOffset") + searchBytes.length;
+ if (fileLength <= nextByteOffset) {
+ nextByteOffset = null;
+ }
+ }
+ ret.putAll(mkGrepResponse(searchBytes, startByteOffset, matches, nextByteOffset));
+ break;
+ }
+ }
+
+ return ret;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * Get the filtered, authorized, sorted log files for a port.
+ */
+ private List<File> logsForPort(String user, File portDir) {
+ try {
+ List<File> workerLogs = DirectoryCleaner.getFilesForDir(portDir).stream()
+ .filter(file -> WORKER_LOG_FILENAME_PATTERN.asPredicate().test(file.getName()))
+ .collect(toList());
+
+ return workerLogs.stream()
+ .filter(log -> resourceAuthorizer.isUserAllowedToAccessFile(user, WorkerLogs.getTopologyPortWorkerLog(log)))
+ .sorted((f1, f2) -> (int) (f2.lastModified() - f1.lastModified()))
+ .collect(toList());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Matched findNMatches(List<File> logs, int numMatches, int fileOffset, int offset, String search) {
+ logs = drop(logs, fileOffset);
+
+ List<Map<String, Object>> matches = new ArrayList<>();
+ int matchCount = 0;
+
+ while (true) {
+ if (logs.isEmpty()) {
+ break;
+ }
+
+ File firstLog = logs.get(0);
+ Map<String, Object> theseMatches;
+ try {
+ LOG.debug("Looking through {}", firstLog);
+ theseMatches = substringSearch(firstLog, search, numMatches - matchCount, offset);
+ } catch (InvalidRequestException e) {
+ LOG.error("Can't search past end of file.", e);
+ theseMatches = new HashMap<>();
+ }
+
+ String fileName = WorkerLogs.getTopologyPortWorkerLog(firstLog);
+
+ List<Map<String, Object>> newMatches = new ArrayList<>(matches);
+ Map<String, Object> currentFileMatch = new HashMap<>(theseMatches);
+ currentFileMatch.put("fileName", fileName);
+ List<String> splitPath;
+ try {
+ splitPath = Arrays.asList(firstLog.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ currentFileMatch.put("port", first(takeLast(splitPath, 2)));
+ newMatches.add(currentFileMatch);
+
+ int newCount = matchCount + ((List<?>)theseMatches.get("matches")).size();
+
+ if (theseMatches.isEmpty()) {
+ // matches and matchCount is not changed
+ logs = rest(logs);
+ offset = 0;
+ fileOffset = fileOffset + 1;
+ } else if (newCount >= numMatches) {
+ matches = newMatches;
+ break;
+ } else {
+ matches = newMatches;
+ logs = rest(logs);
+ offset = 0;
+ fileOffset = fileOffset + 1;
+ matchCount = newCount;
+ }
+ }
+
+ return new Matched(fileOffset, search, matches);
+ }
+
+
+ /**
+ * As the file is read into a buffer, 1/2 the buffer's size at a time, we search the buffer for matches of the
+ * substring and return a list of zero or more matches.
+ */
+ private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, File file, int fileLength, int offsetToBuf,
+ int initBufOffset, BufferedInputStream stream, Integer bytesSkipped,
+ int bytesRead, ByteBuffer haystack, byte[] needle,
+ List<Map<String, Object>> initialMatches, Integer numMatches, byte[] beforeBytes) throws IOException {
+ int bufOffset = initBufOffset;
+ List<Map<String, Object>> matches = initialMatches;
+
+ byte[] newBeforeBytes;
+ Integer newByteOffset;
+
+ while (true) {
+ int offset = offsetOfBytes(haystack.array(), needle, bufOffset);
+ if (matches.size() < numMatches && offset >= 0) {
+ int fileOffset = offsetToBuf + offset;
+ int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length;
+
+ byte[] beforeArg = null;
+ byte[] afterArg = null;
+ if (offset < GREP_CONTEXT_SIZE) {
+ beforeArg = beforeBytes;
+ }
+
+ if (offset > bytesNeededAfterMatch) {
+ afterArg = tryReadAhead(stream, haystack, offset, fileLength, bytesRead);
+ }
+
+ bufOffset = offset + needle.length;
+ matches.add(mkMatchData(needle, haystack, offset, fileOffset,
+ file.getCanonicalPath(), isDaemon, beforeArg, afterArg));
+ } else {
+ int beforeStrToOffset = Math.min(haystack.limit(), GREP_MAX_SEARCH_SIZE);
+ int beforeStrFromOffset = Math.max(0, beforeStrToOffset - GREP_CONTEXT_SIZE);
+ newBeforeBytes = Arrays.copyOfRange(haystack.array(), beforeStrFromOffset, beforeStrToOffset);
+
+ // It's OK if new-byte-offset is negative.
+ // This is normal if we are out of bytes to read from a small file.
+ if (matches.size() >= numMatches) {
+ newByteOffset = ((Number) last(matches).get("byteOffset")).intValue() + needle.length;
+ } else {
+ newByteOffset = bytesSkipped + bytesRead - GREP_MAX_SEARCH_SIZE;
+ }
+
+ break;
+ }
+ }
+
+ return new SubstringSearchResult(matches, newByteOffset, newBeforeBytes);
+ }
+
+ private int rotateGrepBuffer(ByteBuffer buf, BufferedInputStream stream, int totalBytesRead, File file,
+ int fileLength) throws IOException {
+ byte[] bufArray = buf.array();
+
+ // Copy the 2nd half of the buffer to the first half.
+ System.arraycopy(bufArray, GREP_MAX_SEARCH_SIZE, bufArray, 0, GREP_MAX_SEARCH_SIZE);
+
+ // Zero-out the 2nd half to prevent accidental matches.
+ Arrays.fill(bufArray, GREP_MAX_SEARCH_SIZE, bufArray.length, (byte) 0);
+
+ // Fill the 2nd half with new bytes from the stream.
+ int bytesRead = stream.read(bufArray, GREP_MAX_SEARCH_SIZE, Math.min((int) fileLength, GREP_MAX_SEARCH_SIZE));
+ buf.limit(GREP_MAX_SEARCH_SIZE + bytesRead);
+ return totalBytesRead + bytesRead;
+ }
+
+
+ private Map<String, Object> mkMatchData(byte[] needle, ByteBuffer haystack, int haystackOffset, int fileOffset, String fname,
+ boolean isDaemon, byte[] beforeBytes, byte[] afterBytes)
+ throws UnsupportedEncodingException, UnknownHostException {
+ String url;
+ if (isDaemon) {
+ url = urlToMatchCenteredInLogPageDaemonFile(needle, fname, fileOffset, logviewerPort);
+ } else {
+ url = urlToMatchCenteredInLogPage(needle, fname, fileOffset, logviewerPort);
+ }
+
+ byte[] haystackBytes = haystack.array();
+ String beforeString;
+ String afterString;
+
+ if (haystackOffset >= GREP_CONTEXT_SIZE) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(new String(haystackBytes, (haystackOffset - GREP_CONTEXT_SIZE), GREP_CONTEXT_SIZE, "UTF-8"));
+ sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
+ beforeString = sb.toString();
+ } else {
+ int numDesired = Math.max(0, GREP_CONTEXT_SIZE - haystackOffset);
+ int beforeSize = beforeBytes != null ? beforeBytes.length : 0;
+ int numExpected = Math.min(beforeSize, numDesired);
+
+ if (numExpected > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(new String(beforeBytes, numExpected - beforeSize, numExpected, "UTF-8"));
+ sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
+ beforeString = sb.toString();
+ } else {
+ beforeString = new String(haystackBytes, 0, haystackOffset, "UTF-8");
+ }
+ }
+
+ int needleSize = needle.length;
+ int afterOffset = haystackOffset + needleSize;
+ int haystackSize = haystack.limit();
+
+ if ((afterOffset + GREP_CONTEXT_SIZE) < haystackSize) {
+ afterString = new String(haystackBytes, afterOffset, GREP_CONTEXT_SIZE, "UTF-8");
+ } else {
+ int numDesired = GREP_CONTEXT_SIZE - (haystackSize - afterOffset);
+ int afterSize = afterBytes != null ? afterBytes.length : 0;
+ int numExpected = Math.min(afterSize, numDesired);
+
+ if (numExpected > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8"));
+ sb.append(new String(afterBytes, 0, numExpected, "UTF-8"));
+ afterString = sb.toString();
+ } else {
+ afterString = new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8");
+ }
+ }
+
+ Map<String, Object> ret = new HashMap<>();
+ ret.put("byteOffset", fileOffset);
+ ret.put("beforeString", beforeString);
+ ret.put("afterString", afterString);
+ ret.put("matchString", new String(needle, "UTF-8"));
+ ret.put("logviewerURL", url);
+
+ return ret;
+ }
+
+ /**
+ * Tries once to read ahead in the stream to fill the context and
+ * resets the stream to its position before the call.
+ */
+ private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) throws IOException {
+ int numExpected = Math.min(fileLength - bytesRead, GREP_CONTEXT_SIZE);
+ byte[] afterBytes = new byte[numExpected];
+ stream.mark(numExpected);
+ // Only try reading once.
+ stream.read(afterBytes, 0, numExpected);
+ stream.reset();
+ return afterBytes;
+ }
+
+ /**
+ * Searches a given byte array for a match of a sub-array of bytes.
+ * Returns the offset to the byte that matches, or -1 if no match was found.
+ */
+ private int offsetOfBytes(byte[] buffer, byte[] search, int initOffset) {
+ if (search.length <= 0) {
+ throw new IllegalArgumentException("Search array should not be empty.");
+ }
+
+ if (initOffset < 0) {
+ throw new IllegalArgumentException("Start offset shouldn't be negative.");
+ }
+
+ int offset = initOffset;
+ int candidateOffset = initOffset;
+ int valOffset = 0;
+ int retOffset = 0;
+
+ while (true) {
+ if (search.length - valOffset <= 0) {
+ // found
+ retOffset = candidateOffset;
+ break;
+ } else {
+ if (offset >= buffer.length) {
+ // We ran out of buffer for the search.
+ retOffset = -1;
+ break;
+ } else {
+ if (search[valOffset] != buffer[offset]) {
+ // The match at this candidate offset failed, so start over with the
+ // next candidate byte from the buffer.
+ int newOffset = candidateOffset + 1;
+
+ offset = newOffset;
+ candidateOffset = newOffset;
+ valOffset = 0;
+ } else {
+ // So far it matches. Keep going...
+ offset = offset + 1;
+ valOffset = valOffset + 1;
+ }
+ }
+ }
+ }
+
+ return retOffset;
+ }
+
+ /**
+ * This response data only includes a next byte offset if there is more of the file to read.
+ */
+ private Map<String, Object> mkGrepResponse(byte[] searchBytes, Integer offset, List<Map<String, Object>> matches,
+ Integer nextByteOffset) throws UnsupportedEncodingException {
+ Map<String, Object> ret = new HashMap<>();
+ ret.put("searchString", new String(searchBytes, "UTF-8"));
+ ret.put("startByteOffset", offset);
+ ret.put("matches", matches);
+ if (nextByteOffset != null) {
+ ret.put("nextByteOffset", nextByteOffset);
+ }
+ return ret;
+ }
+
+ private String urlToMatchCenteredInLogPage(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
+ String host = Utils.hostname();
+ String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
+ takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 3));
+
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("file", splittedFileName);
+ parameters.put("start", Math.max(0, offset - (DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
+ parameters.put("length", DEFAULT_BYTES_PER_PAGE);
+
+ return URLBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
+ }
+
+ private String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
+ String host = Utils.hostname();
+ String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
+ takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 1));
+
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("file", splittedFileName);
+ parameters.put("start", Math.max(0, offset - (DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2)));
+ parameters.put("length", DEFAULT_BYTES_PER_PAGE);
+
+ return URLBuilder.build(String.format("http://%s:%d/api/v1/daemonlog", host, port), parameters);
+ }
+
+ private static class Matched implements JSONAware {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private int fileOffset;
+ private String searchString;
+ private List<Map<String, Object>> matches;
+
+ public Matched(int fileOffset, String searchString, List<Map<String, Object>> matches) {
+ this.fileOffset = fileOffset;
+ this.searchString = searchString;
+ this.matches = matches;
+ }
+
+ public int getFileOffset() {
+ return fileOffset;
+ }
+
+ public String getSearchString() {
+ return searchString;
+ }
+
+ public List<Map<String, Object>> getMatches() {
+ return matches;
+ }
+
+ @Override
+ public String toJSONString() {
+ try {
+ return OBJECT_MAPPER.writeValueAsString(this);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class SubstringSearchResult {
+ private List<Map<String, Object>> matches;
+ private Integer newByteOffset;
+ private byte[] newBeforeBytes;
+
+ public SubstringSearchResult(List<Map<String, Object>> matches, Integer newByteOffset, byte[] newBeforeBytes) {
+ this.matches = matches;
+ this.newByteOffset = newByteOffset;
+ this.newBeforeBytes = newBeforeBytes;
+ }
+
+ public List<Map<String, Object>> getMatches() {
+ return matches;
+ }
+
+ public Integer getNewByteOffset() {
+ return newByteOffset;
+ }
+
+ public byte[] getNewBeforeBytes() {
+ return newBeforeBytes;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java
new file mode 100644
index 0000000..1cc274c
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerProfileHandler.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.handler;
+
+import j2html.tags.DomContent;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.DirectoryCleaner;
+import org.apache.storm.daemon.wip.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.utils.ServerUtils;
+
+import javax.ws.rs.core.Response;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static j2html.TagCreator.a;
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.head;
+import static j2html.TagCreator.html;
+import static j2html.TagCreator.li;
+import static j2html.TagCreator.link;
+import static j2html.TagCreator.title;
+import static j2html.TagCreator.ul;
+import static java.util.stream.Collectors.toList;
+
+public class LogviewerProfileHandler {
+
+ public static final String WORKER_LOG_FILENAME = "worker.log";
+ private final String logRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+
+ public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer) {
+ this.logRoot = logRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+ }
+
+ public Response listDumpFiles(String topologyId, String hostPort, String user) throws IOException {
+ String portStr = hostPort.split(":")[1];
+ File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
+
+ if (dir.exists()) {
+ String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
+ if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
+ String content = buildDumpFileListPage(topologyId, hostPort, dir);
+ return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+ private String buildDumpFileListPage(String topologyId, String hostPort, File dir) throws IOException {
+ List<DomContent> liTags = getProfilerDumpFiles(dir).stream()
+ .map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + file)))
+ .collect(toList());
+
+ return html(
+ head(
+ title("File Dumps - Storm Log Viewer"),
+ link().withRel("stylesheet").withHref("/css/bootstrap-3.3.1.min.css"),
+ link().withRel("stylesheet").withHref("/css/jquery.dataTables.1.10.4.min.css"),
+ link().withRel("stylesheet").withHref("/css/style.css")
+ ),
+ body(
+ ul(liTags.toArray(new DomContent[]{}))
+ )
+ ).render();
+ }
+
+ public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException {
+ String portStr = hostPort.split(":")[1];
+ File dir = new File(String.join(ServerUtils.FILE_PATH_SEPARATOR, logRoot, topologyId, portStr));
+ File file = new File(dir, fileName);
+
+ if (dir.exists() && file.exists()) {
+ String workerFileRelativePath = String.join(ServerUtils.FILE_PATH_SEPARATOR, topologyId, portStr, WORKER_LOG_FILENAME);
+ if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) {
+ return LogviewerResponseBuilder.buildDownloadFile(file);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+ private List<String> getProfilerDumpFiles(File dir) throws IOException {
+ List<File> filesForDir = DirectoryCleaner.getFilesForDir(dir);
+ return filesForDir.stream().filter(file -> {
+ String fileName = file.getName();
+ return StringUtils.isNotEmpty(fileName)
+ && (fileName.endsWith(".txt") || fileName.endsWith(".jfr") || fileName.endsWith(".bin"));
+ }).map(File::getName).collect(toList());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java
new file mode 100644
index 0000000..6aa1050
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.utils;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.storm.StormTimer;
+import org.apache.storm.daemon.DirectoryCleaner;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+import org.jooq.lambda.tuple.Tuple2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.function.BinaryOperator;
+import java.util.stream.StreamSupport;
+
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB;
+import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB;
+
+public class LogCleaner implements Runnable, Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
+ public static final String WORKER_YAML = "worker.yaml";
+
+ private final Map<String, Object> stormConf;
+ private final Integer intervalSecs;
+ private final String logRootDir;
+ private StormTimer logviewerCleanupTimer;
+ private final long maxSumWorkerLogsSizeMb;
+ private long maxPerWorkerLogsSizeMb;
+
+ public LogCleaner(Map<String, Object> stormConf) {
+ String logRootDir = ConfigUtils.workerArtifactsRoot(stormConf);
+
+ this.stormConf = stormConf;
+ this.intervalSecs = ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
+ this.logRootDir = logRootDir;
+
+ maxSumWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB));
+ maxPerWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB));
+ maxPerWorkerLogsSizeMb = Math.min(maxPerWorkerLogsSizeMb, (long) (maxSumWorkerLogsSizeMb * 0.5));
+
+ LOG.info("configured max total size of worker logs: {} MB, max total size of worker logs per directory: {} MB",
+ maxSumWorkerLogsSizeMb, maxPerWorkerLogsSizeMb);
+ }
+
+ public void start() {
+ if (intervalSecs != null) {
+ LOG.debug("starting log cleanup thread at interval: {}", intervalSecs);
+
+ logviewerCleanupTimer = new StormTimer("logviewer-cleanup", (t, e) -> {
+ LOG.error("Error when doing logs cleanup", e);
+ Utils.exitProcess(20, "Error when doing log cleanup");
+ });
+
+ logviewerCleanupTimer.scheduleRecurring(0, intervalSecs, this);
+ } else {
+ LOG.warn("The interval for log cleanup is not set. Skip starting log cleanup thread.");
+ }
+
+ }
+
+ public void close() {
+ if (logviewerCleanupTimer != null) {
+ try {
+ logviewerCleanupTimer.close();
+ } catch (Exception ex) {
+ throw Utils.wrapInRuntime(ex);
+ }
+ }
+ }
+
+ /**
+ * Delete old log dirs for which the workers are no longer alive.
+ */
+ public void run() {
+ try {
+ int nowSecs = Time.currentTimeSecs();
+ Set<File> oldLogDirs = selectDirsForCleanup(nowSecs * 1000, logRootDir);
+
+ DirectoryCleaner cleaner = new DirectoryCleaner();
+ SortedSet<File> deadWorkerDirs = getDeadWorkerDirs(nowSecs, oldLogDirs);
+
+ LOG.debug("log cleanup: now={} old log dirs {} dead worker dirs {}", nowSecs,
+ oldLogDirs.stream().map(File::getName).collect(joining(",")),
+ deadWorkerDirs.stream().map(File::getName).collect(joining(",")));
+
+ deadWorkerDirs.forEach(Unchecked.consumer(dir -> {
+ String path = dir.getCanonicalPath();
+ LOG.info("Cleaning up: Removing {}", path);
+
+ try {
+ Utils.forceDelete(path);
+ cleanupEmptyTopoDirectory(dir);
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ }
+ }));
+
+ perWorkerDirCleanup(new File(logRootDir), maxPerWorkerLogsSizeMb * 1024 * 1024, cleaner);
+ globalLogCleanup(new File(logRootDir), maxSumWorkerLogsSizeMb * 1024 * 1024, cleaner);
+ } catch (Exception ex) {
+ LOG.error("Exception while cleaning up old log.", ex);
+ }
+ }
+
+ /**
+ * Delete the oldest files in each overloaded worker log dir.
+ */
+ private void perWorkerDirCleanup(File rootDir, long size, DirectoryCleaner cleaner) {
+ WorkerLogs.getAllWorkerDirs(rootDir).forEach(Unchecked.consumer(dir -> {
+ cleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null);
+ }));
+ }
+
+ /**
+ * Delete the oldest files in overloaded worker-artifacts globally.
+ */
+ private void globalLogCleanup(File rootDir, long size, DirectoryCleaner cleaner) throws Exception {
+ List<File> workerDirs = new ArrayList<>(WorkerLogs.getAllWorkerDirs(rootDir));
+ Set<String> aliveWorkerDirs = new HashSet<>(getAliveWorkerDirs(rootDir));
+
+ cleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs);
+ }
+
+ /**
+ * Return a sorted set of java.io.Files that were written by workers that are now active.
+ */
+ private SortedSet<String> getAliveWorkerDirs(File rootDir) throws Exception {
+ Set<String> aliveIds = getAliveIds(Time.currentTimeSecs());
+ Set<File> logDirs = WorkerLogs.getAllWorkerDirs(rootDir);
+ Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
+
+ return idToDir.entrySet().stream()
+ .filter(entry -> aliveIds.contains(entry.getKey()))
+ .map(Unchecked.function(entry -> entry.getValue().getCanonicalPath()))
+ .collect(toCollection(TreeSet::new));
+ }
+
+ private Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
+ return logDirs.stream().map(Unchecked.function(logDir -> {
+ Optional<File> metaFile = getMetadataFileForWorkerLogDir(logDir);
+
+ return metaFile.map(Unchecked.function(m -> new Tuple2<>(getWorkerIdFromMetadataFile(m.getCanonicalPath()), logDir)))
+ .orElse(new Tuple2<>("", logDir));
+ })).collect(toMap(Tuple2::v1, Tuple2::v2));
+ }
+
+ private Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
+ File metaFile = new File(logDir, WORKER_YAML);
+ if (metaFile.exists()) {
+ return Optional.of(metaFile);
+ } else {
+ LOG.warn("Could not find {} to clean up for {}", metaFile.getCanonicalPath(), logDir);
+ return Optional.empty();
+ }
+ }
+
+ private String getWorkerIdFromMetadataFile(String metaFile) {
+ Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
+ return ObjectReader.getString(map.get("worker-id"), null);
+ }
+
+ private Set<String> getAliveIds(int nowSecs) throws Exception {
+ return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
+ .filter(entry -> Objects.nonNull(entry.getValue())
+ && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf))
+ .map(Map.Entry::getKey)
+ .collect(toCollection(TreeSet::new));
+ }
+
+ /**
+ * Delete the topo dir if it contains zero port dirs.
+ */
+ private void cleanupEmptyTopoDirectory(File dir) throws IOException {
+ File topoDir = dir.getParentFile();
+ if (topoDir.listFiles().length == 0) {
+ Utils.forceDelete(topoDir.getCanonicalPath());
+ }
+ }
+
+ /**
+ * Return a sorted set of java.io.Files that were written by workers that are now dead.
+ */
+ private SortedSet<File> getDeadWorkerDirs(int nowSecs, Set<File> logDirs) throws Exception {
+ if (logDirs.isEmpty()) {
+ return new TreeSet<>();
+ } else {
+ Set<String> aliveIds = getAliveIds(nowSecs);
+ Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
+
+ return idToDir.entrySet().stream()
+ .filter(entry -> !aliveIds.contains(entry.getKey()))
+ .map(Map.Entry::getValue)
+ .collect(toCollection(TreeSet::new));
+ }
+ }
+
+ private Set<File> selectDirsForCleanup(int nowMillis, String rootDir) {
+ FileFilter fileFilter = mkFileFilterForLogCleanup(nowMillis);
+
+ return Arrays.stream(new File(rootDir).listFiles())
+ .flatMap(topoDir -> Arrays.stream(topoDir.listFiles(fileFilter)))
+ .collect(toCollection(TreeSet::new));
+ }
+
+ private FileFilter mkFileFilterForLogCleanup(int nowMillis) {
+ final int cutoffAgeMillis = cleanupCutoffAgeMillis(nowMillis);
+ return file -> !file.isFile() && lastModifiedTimeWorkerLogdir(file) <= cutoffAgeMillis;
+ }
+
+ /**
+ * Return the last modified time for all log files in a worker's log dir.
+ * Using stream rather than File.listFiles is to avoid large mem usage
+ * when a directory has too many files.
+ */
+ private long lastModifiedTimeWorkerLogdir(File logDir) {
+ Optional<DirectoryStream<Path>> dirStreamOptional = getStreamForDir(logDir);
+ long dirModified = logDir.lastModified();
+
+ if (!dirStreamOptional.isPresent()) {
+ return dirModified;
+ }
+
+ DirectoryStream<Path> dirStream = dirStreamOptional.get();
+ try {
+ return StreamSupport.stream(dirStream.spliterator(), false)
+ .reduce(dirModified, (maximum, path) -> {
+ long curr = path.toFile().lastModified();
+ return curr > maximum ? curr : maximum;
+ }, BinaryOperator.maxBy(Long::compareTo));
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ return dirModified;
+ } finally {
+ if (DirectoryStream.class.isInstance(dirStream)) {
+ IOUtils.closeQuietly(dirStream);
+ }
+ }
+ }
+
+ private Optional<DirectoryStream<Path>> getStreamForDir(File file) {
+ try {
+ return Optional.of(Files.newDirectoryStream(file.toPath()));
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ return Optional.empty();
+ }
+ }
+
+ private int cleanupCutoffAgeMillis(int nowMillis) {
+ return nowMillis - (ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_AGE_MINS)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java
new file mode 100644
index 0000000..dcb2e2c
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.utils;
+
+import javax.ws.rs.core.Response;
+import java.io.File;
+import java.io.IOException;
+
+public class LogFileDownloader {
+
+ private final String logRoot;
+ private final String daemonLogRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+
+ public LogFileDownloader(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
+ this.logRoot = logRoot;
+ this.daemonLogRoot = daemonLogRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+ }
+
+ public Response downloadFile(String fileName, String user, boolean isDaemon) throws IOException {
+ String rootDir = isDaemon ? daemonLogRoot : logRoot;
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ if (file.exists()) {
+ if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ return LogviewerResponseBuilder.buildDownloadFile(file);
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java
new file mode 100644
index 0000000..bc60e09
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.utils;
+
+import com.google.common.io.ByteStreams;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.ui.UIHelpers;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import static j2html.TagCreator.body;
+import static j2html.TagCreator.h2;
+import static javax.ws.rs.core.Response.Status.OK;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+
+public class LogviewerResponseBuilder {
+
+ private LogviewerResponseBuilder() {
+ }
+
+ public static Response buildSuccessHtmlResponse(String content) {
+ return Response.status(OK).entity(content)
+ .type(MediaType.TEXT_HTML_TYPE).build();
+ }
+
+ public static Response buildSuccessJsonResponse(Object entity, String callback, String origin) {
+ return new JsonResponseBuilder().setData(entity).setCallback(callback)
+ .setHeaders(LogviewerResponseBuilder.getHeadersForSuccessResponse(origin)).build();
+ }
+
+ public static Response buildDownloadFile(File file) throws IOException {
+ // do not close this InputStream in method: it will be used from jetty server
+ InputStream is = new FileInputStream(file);
+ return Response.status(OK)
+ .entity(wrapWithStreamingOutput(is))
+ .type(MediaType.APPLICATION_OCTET_STREAM_TYPE)
+ .header("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"")
+ .build();
+ }
+
+ public static Response buildResponseUnautohrizedUser(String user) {
+ String entity = buildUnauthorizedUserHtml(user);
+ return Response.status(OK)
+ .entity(entity)
+ .type(MediaType.TEXT_HTML_TYPE)
+ .build();
+ }
+
+ public static Response buildResponsePageNotFound() {
+ return Response.status(404)
+ .entity("Page not found")
+ .type(MediaType.TEXT_HTML_TYPE)
+ .build();
+ }
+
+ public static Response buildUnauthorizedUserJsonResponse(String user, String callback) {
+ return new JsonResponseBuilder().setData(UIHelpers.unauthorizedUserJson(user))
+ .setCallback(callback).setStatus(401).build();
+ }
+
+ public static Response buildExceptionJsonResponse(Exception ex, String callback) {
+ return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(ex))
+ .setCallback(callback).setStatus(500).build();
+ }
+
+ private static Map<String, Object> getHeadersForSuccessResponse(String origin) {
+ Map<String, Object> headers = new HashMap<>();
+ headers.put("Access-Control-Allow-Origin", origin);
+ headers.put("Access-Control-Allow-Credentials", "true");
+ return headers;
+ }
+
+ private static String buildUnauthorizedUserHtml(String user) {
+ String content = "User '" + escapeHtml(user) + "' is not authorized.";
+ return body(h2(content)).render();
+ }
+
+ private static StreamingOutput wrapWithStreamingOutput(final InputStream inputStream) {
+ return os -> {
+ OutputStream wrappedOutputStream = os;
+ if (!(os instanceof BufferedOutputStream)) {
+ wrappedOutputStream = new BufferedOutputStream(os);
+ }
+
+ ByteStreams.copy(inputStream, wrappedOutputStream);
+
+ wrappedOutputStream.flush();
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java
new file mode 100644
index 0000000..c7a45e7
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.utils;
+
+import com.google.common.collect.Sets;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.security.auth.AuthUtils;
+import org.apache.storm.security.auth.IGroupMappingServiceProvider;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerConfigUtils;
+import org.apache.storm.utils.Utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.storm.DaemonConfig.UI_FILTER;
+
+public class ResourceAuthorizer {
+
+ private final Map<String, Object> stormConf;
+ private final IGroupMappingServiceProvider groupMappingServiceProvider;
+
+ public ResourceAuthorizer(Map<String, Object> stormConf) {
+ this.stormConf = stormConf;
+ this.groupMappingServiceProvider = AuthUtils.GetGroupMappingServiceProviderPlugin(stormConf);
+ }
+
+ public boolean isUserAllowedToAccessFile(String fileName, String user) {
+ return isUiFilterNotSet() || isAuthorizedLogUser(user, fileName);
+ }
+
+ private boolean isUiFilterNotSet() {
+ return StringUtils.isBlank(ObjectReader.getString(stormConf.get(UI_FILTER), null));
+ }
+
+ private boolean isAuthorizedLogUser(String user, String fileName) {
+ if (StringUtils.isEmpty(user) || StringUtils.isEmpty(fileName)
+ || getLogUserGroupWhitelist(fileName) == null) {
+ return false;
+ } else {
+ Set<String> groups = getUserGroups(user);
+ LogUserGroupWhitelist whitelist = getLogUserGroupWhitelist(fileName);
+
+ List<String> logsUsers = new ArrayList<>();
+ logsUsers.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_USERS)));
+ logsUsers.addAll(ObjectReader.getStrings(stormConf.get(Config.NIMBUS_ADMINS)));
+ logsUsers.addAll(whitelist.getUserWhitelist());
+
+ List<String> logsGroups = new ArrayList<>();
+ logsGroups.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_GROUPS)));
+ logsGroups.addAll(whitelist.getGroupWhitelist());
+
+ return logsUsers.stream().anyMatch(u -> u.equals(user)) ||
+ Sets.intersection(groups, new HashSet<>(logsGroups)).size() < 0;
+
+ }
+ }
+
+ public LogUserGroupWhitelist getLogUserGroupWhitelist(String fileName) {
+ File wlFile = ServerConfigUtils.getLogMetaDataFile(fileName);
+ Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(wlFile.getAbsolutePath());
+
+ if (map == null) {
+ return null;
+ }
+
+ List<String> logsUsers = ObjectReader.getStrings(map.get(DaemonConfig.LOGS_USERS));
+ List<String> logsGroups = ObjectReader.getStrings(map.get(DaemonConfig.LOGS_GROUPS));
+ return new LogUserGroupWhitelist(
+ logsUsers.isEmpty() ? new HashSet<>() : new HashSet<>(logsUsers),
+ logsGroups.isEmpty() ? new HashSet<>() : new HashSet<>(logsGroups)
+ );
+ }
+
+ private Set<String> getUserGroups(String user) {
+ try {
+ if (StringUtils.isEmpty(user)) {
+ return new HashSet<>();
+ } else {
+ return groupMappingServiceProvider.getGroups(user);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static class LogUserGroupWhitelist {
+
+ private Set<String> userWhitelist;
+ private Set<String> groupWhitelist;
+
+ public LogUserGroupWhitelist(Set<String> userWhitelist, Set<String> groupWhitelist) {
+ this.userWhitelist = userWhitelist;
+ this.groupWhitelist = groupWhitelist;
+ }
+
+ public Set<String> getUserWhitelist() {
+ return userWhitelist;
+ }
+
+ public Set<String> getGroupWhitelist() {
+ return groupWhitelist;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java
new file mode 100644
index 0000000..c49a4a1
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java
@@ -0,0 +1,63 @@
+package org.apache.storm.daemon.wip.logviewer.utils;
+
+import org.apache.storm.daemon.DirectoryCleaner;
+import org.apache.storm.utils.Utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toCollection;
+import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
+
+public class WorkerLogs {
+
+ private WorkerLogs() {
+ }
+
+ /**
+ * Return the path of the worker log with the format of topoId/port/worker.log.*
+ */
+ public static String getTopologyPortWorkerLog(File file) {
+ try {
+ String[] splitted = file.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR);
+ List<String> split = takeLast(Arrays.asList(splitted), 3);
+
+ return String.join(Utils.FILE_PATH_SEPARATOR, split);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static List<File> getAllLogsForRootDir(File logDir) throws IOException {
+ List<File> files = new ArrayList<>();
+ Set<File> topoDirFiles = getAllWorkerDirs(logDir);
+ if (topoDirFiles != null) {
+ for (File portDir : topoDirFiles) {
+ files.addAll(DirectoryCleaner.getFilesForDir(portDir));
+ }
+ }
+
+ return files;
+ }
+
+ public static Set<File> getAllWorkerDirs(File rootDir) {
+ File[] rootDirFiles = rootDir.listFiles();
+ if (rootDirFiles != null) {
+ return Arrays.stream(rootDirFiles).flatMap(topoDir -> {
+ File[] topoFiles = topoDir.listFiles();
+ return topoFiles != null ? Arrays.stream(topoFiles) : Stream.empty();
+ }).collect(toCollection(TreeSet::new));
+ }
+
+ return new TreeSet<>();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java
new file mode 100644
index 0000000..c1b4bc7
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.webapp;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
+import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogDownloadHandler;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogPageHandler;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogSearchHandler;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerProfileHandler;
+import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.security.auth.AuthUtils;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ObjectReader;
+
+import javax.ws.rs.ApplicationPath;
+import javax.ws.rs.core.Application;
+import java.io.File;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME;
+
+@ApplicationPath("")
+public class LogviewerApplication extends Application {
+ private static Map<String, Object> stormConf;
+ private final Set<Object> singletons = new HashSet<Object>();
+
+ /**
+ * Constructor.
+ */
+ public LogviewerApplication() {
+ String logRoot = ConfigUtils.workerArtifactsRoot(stormConf);
+ String daemonLogRoot = logRootDir(ObjectReader.getString(stormConf.get(LOGVIEWER_APPENDER_NAME)));
+
+ ResourceAuthorizer resourceAuthorizer = new ResourceAuthorizer(stormConf);
+
+ LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, resourceAuthorizer);
+ LogviewerProfileHandler profileHandler = new LogviewerProfileHandler(logRoot, resourceAuthorizer);
+ LogviewerLogDownloadHandler logDownloadHandler = new LogviewerLogDownloadHandler(logRoot, daemonLogRoot,
+ resourceAuthorizer);
+ LogviewerLogSearchHandler logSearchHandler = new LogviewerLogSearchHandler(stormConf, logRoot, daemonLogRoot,
+ resourceAuthorizer);
+ IHttpCredentialsPlugin httpCredsHandler = AuthUtils.GetUiHttpCredentialsPlugin(stormConf);
+
+ singletons.add(new LogviewerResource(logviewer, profileHandler, logDownloadHandler, logSearchHandler, httpCredsHandler));
+ singletons.add(new AuthorizationExceptionMapper());
+ }
+
+ @Override
+ public Set<Object> getSingletons() {
+ return singletons;
+ }
+
+ public static void setup(Map<String, Object> stormConf) {
+ LogviewerApplication.stormConf = stormConf;
+ }
+
+ /**
+ * Given an appender name, as configured, get the parent directory of the appender's log file.
+ * Note that if anything goes wrong, this will throw an Error and exit.
+ */
+ private String logRootDir(String appenderName) {
+ Appender appender = ((LoggerContext) LogManager.getContext()).getConfiguration().getAppender(appenderName);
+ if (appenderName != null && appender != null && RollingFileAppender.class.isInstance(appender)) {
+ return new File(((RollingFileAppender) appender).getFileName()).getParent();
+ } else {
+ throw new RuntimeException("Log viewer could not find configured appender, or the appender is not a FileAppender. "
+ + "Please check that the appender name configured in storm and log4j agree.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java
new file mode 100644
index 0000000..fd8ea97
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.webapp;
+
+import com.codahale.metrics.Meter;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.common.JsonResponseBuilder;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogDownloadHandler;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogPageHandler;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogSearchHandler;
+import org.apache.storm.daemon.wip.logviewer.handler.LogviewerProfileHandler;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.ui.UIHelpers;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.Map;
+
+@Path("/")
+public class LogviewerResource {
+ private static final Logger LOG = LoggerFactory.getLogger(LogviewerResource.class);
+
+ private static final Meter meterLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-log-page-http-requests");
+ private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-daemonlog-page-http-requests");
+ private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-file-http-requests");
+ private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-daemon-file-http-requests");
+ private static final Meter meterListLogsHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-list-logs-http-requests");
+
+ private final LogviewerLogPageHandler logviewer;
+ private final LogviewerProfileHandler profileHandler;
+ private final LogviewerLogDownloadHandler logDownloadHandler;
+ private final LogviewerLogSearchHandler logSearchHandler;
+ private final IHttpCredentialsPlugin httpCredsHandler;
+
+ public LogviewerResource(LogviewerLogPageHandler logviewerParam, LogviewerProfileHandler profileHandler,
+ LogviewerLogDownloadHandler logDownloadHandler, LogviewerLogSearchHandler logSearchHandler,
+ IHttpCredentialsPlugin httpCredsHandler) {
+ this.logviewer = logviewerParam;
+ this.profileHandler = profileHandler;
+ this.logDownloadHandler = logDownloadHandler;
+ this.logSearchHandler = logSearchHandler;
+ this.httpCredsHandler = httpCredsHandler;
+ }
+
+ @GET
+ @Path("/log")
+ public Response log(@Context HttpServletRequest request) throws IOException {
+ meterLogPageHttpRequests.mark();
+
+ try {
+ String user = httpCredsHandler.getUserName(request);
+ Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null;
+ Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null;
+ String decodedFileName = URLDecoder.decode(request.getParameter("file"));
+ String grep = request.getParameter("grep");
+ return logviewer.logPage(decodedFileName, start, length, grep, user);
+ } catch (InvalidRequestException e) {
+ LOG.error(e.getMessage(), e);
+ return Response.status(400).entity(e.getMessage()).build();
+ }
+ }
+
+ @GET
+ @Path("/daemonlog")
+ public Response daemonLog(@Context HttpServletRequest request) throws IOException {
+ meterDaemonLogPageHttpRequests.mark();
+
+ try {
+ String user = httpCredsHandler.getUserName(request);
+ Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null;
+ Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null;
+ String decodedFileName = URLDecoder.decode(request.getParameter("file"));
+ String grep = request.getParameter("grep");
+ return logviewer.daemonLogPage(decodedFileName, start, length, grep, user);
+ } catch (InvalidRequestException e) {
+ LOG.error(e.getMessage(), e);
+ return Response.status(400).entity(e.getMessage()).build();
+ }
+ }
+
+ @GET
+ @Path("/searchLogs")
+ public Response searchLogs(@Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ String topologyId = request.getParameter("topoId");
+ String portStr = request.getParameter("port");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
+ }
+
+ @GET
+ @Path("/listLogs")
+ public Response listLogs(@Context HttpServletRequest request) throws IOException {
+ meterListLogsHttpRequests.mark();
+
+ String user = httpCredsHandler.getUserName(request);
+ String topologyId = request.getParameter("topoId");
+ String portStr = request.getParameter("port");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
+ }
+
+ @GET
+ @Path("/dumps/{topo-id}/{host-port}")
+ public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
+ @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ return profileHandler.listDumpFiles(topologyId, hostPort, user);
+ }
+
+ @GET
+ @Path("/dumps/{topo-id}/{host-port}/{filename}")
+ public Response downloadDumpFile(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
+ @PathParam("filename") String fileName, @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ return profileHandler.downloadDumpFile(topologyId, hostPort, fileName, user);
+ }
+
+ @GET
+ @Path("/download")
+ public Response downloadLogFile(@Context HttpServletRequest request) throws IOException {
+ meterDownloadLogFileHttpRequests.mark();
+
+ String user = httpCredsHandler.getUserName(request);
+ String file = request.getParameter("file");
+ String decodedFileName = URLDecoder.decode(file);
+ return logDownloadHandler.downloadLogFile(decodedFileName, user);
+ }
+
+ @GET
+ @Path("/daemondownload")
+ public Response downloadDaemonLogFile(@Context HttpServletRequest request) throws IOException {
+ meterDownloadLogDaemonFileHttpRequests.mark();
+
+ String user = httpCredsHandler.getUserName(request);
+ String file = request.getParameter("file");
+ String decodedFileName = URLDecoder.decode(file);
+ return logDownloadHandler.downloadDaemonLogFile(decodedFileName, user);
+ }
+
+ @GET
+ @Path("/search/{file}")
+ public Response search(@PathParam("file") String file, @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ boolean isDaemon = StringUtils.equals(request.getParameter("is-daemon"), "yes");
+ String decodedFileName = URLDecoder.decode(file);
+ String searchString = request.getParameter("search-string");
+ String numMatchesStr = request.getParameter("num-matches");
+ String startByteOffset = request.getParameter("start-byte-offset");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ try {
+ return logSearchHandler.searchLogFile(decodedFileName, user, isDaemon, searchString, numMatchesStr,
+ startByteOffset, callback, origin);
+ } catch (InvalidRequestException e) {
+ LOG.error(e.getMessage(), e);
+ return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(e)).setCallback(callback)
+ .setStatus(400).build();
+ }
+ }
+
+ @GET
+ @Path("/deepSearch/{topoId}")
+ public Response deepSearch(@PathParam("topoId") String topologyId,
+ @Context HttpServletRequest request) throws IOException {
+ String user = httpCredsHandler.getUserName(request);
+ String searchString = request.getParameter("search-string");
+ String numMatchesStr = request.getParameter("num-matches");
+ String portStr = request.getParameter("port");
+ String startFileOffset = request.getParameter("start-file-offset");
+ String startByteOffset = request.getParameter("start-byte-offset");
+ String searchArchived = request.getParameter("search-archived");
+ String callback = request.getParameter("callback");
+ String origin = request.getHeader("Origin");
+
+ return logSearchHandler.deepSearchLogsForTopology(topologyId, user, searchString, numMatchesStr, portStr,
+ startFileOffset, startByteOffset, BooleanUtils.toBooleanObject(searchArchived), callback, origin);
+ }
+
+ private int parseIntegerFromMap(Map map, String parameterKey) throws InvalidRequestException {
+ try {
+ return Integer.parseInt(((String[]) map.get(parameterKey))[0]);
+ } catch (NumberFormatException ex) {
+ throw new InvalidRequestException("Could not make an integer out of the query parameter '"
+ + parameterKey + "'", ex);
+ }
+ }
+
+}
[11/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* also fixed failing tests
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7f846915
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7f846915
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7f846915
Branch: refs/heads/master
Commit: 7f84691516dbbea28a780acb05bc5bae449f67cd
Parents: bb5c247
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Jul 14 14:24:20 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Jul 14 14:24:20 2017 +0900
----------------------------------------------------------------------
.../logviewer/handler/LogviewerLogSearchHandler.java | 10 ++++------
.../logviewer/handler/LogviewerLogSearchHandlerTest.java | 11 ++---------
2 files changed, 6 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7f846915/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
index 7d800e7..304c2d6 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
@@ -490,10 +490,7 @@ public class LogviewerLogSearchHandler {
String afterString;
if (haystackOffset >= GREP_CONTEXT_SIZE) {
- StringBuilder sb = new StringBuilder();
- sb.append(new String(haystackBytes, (haystackOffset - GREP_CONTEXT_SIZE), GREP_CONTEXT_SIZE, "UTF-8"));
- sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
- beforeString = sb.toString();
+ beforeString = new String(haystackBytes, (haystackOffset - GREP_CONTEXT_SIZE), GREP_CONTEXT_SIZE, "UTF-8");
} else {
int numDesired = Math.max(0, GREP_CONTEXT_SIZE - haystackOffset);
int beforeSize = beforeBytes != null ? beforeBytes.length : 0;
@@ -501,7 +498,7 @@ public class LogviewerLogSearchHandler {
if (numExpected > 0) {
StringBuilder sb = new StringBuilder();
- sb.append(new String(beforeBytes, numExpected - beforeSize, numExpected, "UTF-8"));
+ sb.append(new String(beforeBytes, beforeSize - numExpected, numExpected, "UTF-8"));
sb.append(new String(haystackBytes, 0, haystackOffset, "UTF-8"));
beforeString = sb.toString();
} else {
@@ -632,7 +629,8 @@ public class LogviewerLogSearchHandler {
return URLBuilder.build(String.format("http://%s:%d/api/v1/log", host, port), parameters);
}
- private String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
+ @VisibleForTesting
+ String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, String fname, int offset, Integer port) throws UnknownHostException {
String host = Utils.hostname();
String splittedFileName = String.join(Utils.FILE_PATH_SEPARATOR,
takeLast(Arrays.asList(fname.split(Utils.FILE_PATH_SEPARATOR)), 1));
http://git-wip-us.apache.org/repos/asf/storm/blob/7f846915/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
index 29dff1b..9e36b5a 100644
--- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java
@@ -117,7 +117,7 @@ public class LogviewerLogSearchHandlerTest {
when(mockedUtil.hostname()).thenReturn(expectedHost);
- String actualUrl = handler.urlToMatchCenteredInLogPage(new byte[42], expectedFname, 27526, 8888);
+ String actualUrl = handler.urlToMatchCenteredInLogPageDaemonFile(new byte[42], expectedFname, 27526, 8888);
assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/daemonlog?file=" + expectedFname +
"&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl);
@@ -167,7 +167,6 @@ public class LogviewerLogSearchHandlerTest {
"/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
));
- // FIXME: currently fail on this...
matches.add(buildMatchData(134,
" needle000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000needle ",
"\n",
@@ -255,7 +254,6 @@ public class LogviewerLogSearchHandlerTest {
}
}
- // FIXME: failing now
@Test
public void testNoOffsetReturnedWhenFileEndsOnBufferOffset() throws InvalidRequestException, UnknownHostException {
Utils prevUtils = null;
@@ -339,7 +337,6 @@ public class LogviewerLogSearchHandlerTest {
expected.put("startByteOffset", 0);
expected.put("nextByteOffset", 6252);
- // FIXME: currently failing on below...
List<Map<String, Object>> matches = new ArrayList<>();
matches.add(buildMatchData(5,
@@ -399,7 +396,6 @@ public class LogviewerLogSearchHandlerTest {
}
}
- // FIXME: this is failing...
@Test
public void testCorrectMatchOffsetIsReturnedWhenSkippingBytes() throws InvalidRequestException, UnknownHostException {
Utils prevUtils = null;
@@ -440,8 +436,6 @@ public class LogviewerLogSearchHandlerTest {
}
}
- // FIXME: this is failing...
- // java.lang.StringIndexOutOfBoundsException: String index out of range: -39
@Test
public void testAnotherPatterns1() throws UnknownHostException, InvalidRequestException {
Utils prevUtils = null;
@@ -481,7 +475,7 @@ public class LogviewerLogSearchHandlerTest {
"/api/v1/log?file=test%2Fresources%2F" + file.getName() + "&start=0&length=51200"
));
- expected.put("matches", Collections.emptyList());
+ expected.put("matches", matches);
assertEquals(expected, searchResult);
} finally {
@@ -489,7 +483,6 @@ public class LogviewerLogSearchHandlerTest {
}
}
- // FIXME: this is failing...
@Test
public void testAnotherPatterns2() throws UnknownHostException, InvalidRequestException {
Utils prevUtils = null;
[04/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java
deleted file mode 100644
index 6aa1050..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogCleaner.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.utils;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.storm.StormTimer;
-import org.apache.storm.daemon.DirectoryCleaner;
-import org.apache.storm.daemon.supervisor.SupervisorUtils;
-import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Utils;
-import org.jooq.lambda.Unchecked;
-import org.jooq.lambda.tuple.Tuple2;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.function.BinaryOperator;
-import java.util.stream.StreamSupport;
-
-import static java.util.stream.Collectors.joining;
-import static java.util.stream.Collectors.toCollection;
-import static java.util.stream.Collectors.toMap;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB;
-import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB;
-
-public class LogCleaner implements Runnable, Closeable {
- private static final Logger LOG = LoggerFactory.getLogger(LogCleaner.class);
- public static final String WORKER_YAML = "worker.yaml";
-
- private final Map<String, Object> stormConf;
- private final Integer intervalSecs;
- private final String logRootDir;
- private StormTimer logviewerCleanupTimer;
- private final long maxSumWorkerLogsSizeMb;
- private long maxPerWorkerLogsSizeMb;
-
- public LogCleaner(Map<String, Object> stormConf) {
- String logRootDir = ConfigUtils.workerArtifactsRoot(stormConf);
-
- this.stormConf = stormConf;
- this.intervalSecs = ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
- this.logRootDir = logRootDir;
-
- maxSumWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB));
- maxPerWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB));
- maxPerWorkerLogsSizeMb = Math.min(maxPerWorkerLogsSizeMb, (long) (maxSumWorkerLogsSizeMb * 0.5));
-
- LOG.info("configured max total size of worker logs: {} MB, max total size of worker logs per directory: {} MB",
- maxSumWorkerLogsSizeMb, maxPerWorkerLogsSizeMb);
- }
-
- public void start() {
- if (intervalSecs != null) {
- LOG.debug("starting log cleanup thread at interval: {}", intervalSecs);
-
- logviewerCleanupTimer = new StormTimer("logviewer-cleanup", (t, e) -> {
- LOG.error("Error when doing logs cleanup", e);
- Utils.exitProcess(20, "Error when doing log cleanup");
- });
-
- logviewerCleanupTimer.scheduleRecurring(0, intervalSecs, this);
- } else {
- LOG.warn("The interval for log cleanup is not set. Skip starting log cleanup thread.");
- }
-
- }
-
- public void close() {
- if (logviewerCleanupTimer != null) {
- try {
- logviewerCleanupTimer.close();
- } catch (Exception ex) {
- throw Utils.wrapInRuntime(ex);
- }
- }
- }
-
- /**
- * Delete old log dirs for which the workers are no longer alive.
- */
- public void run() {
- try {
- int nowSecs = Time.currentTimeSecs();
- Set<File> oldLogDirs = selectDirsForCleanup(nowSecs * 1000, logRootDir);
-
- DirectoryCleaner cleaner = new DirectoryCleaner();
- SortedSet<File> deadWorkerDirs = getDeadWorkerDirs(nowSecs, oldLogDirs);
-
- LOG.debug("log cleanup: now={} old log dirs {} dead worker dirs {}", nowSecs,
- oldLogDirs.stream().map(File::getName).collect(joining(",")),
- deadWorkerDirs.stream().map(File::getName).collect(joining(",")));
-
- deadWorkerDirs.forEach(Unchecked.consumer(dir -> {
- String path = dir.getCanonicalPath();
- LOG.info("Cleaning up: Removing {}", path);
-
- try {
- Utils.forceDelete(path);
- cleanupEmptyTopoDirectory(dir);
- } catch (Exception ex) {
- LOG.error(ex.getMessage(), ex);
- }
- }));
-
- perWorkerDirCleanup(new File(logRootDir), maxPerWorkerLogsSizeMb * 1024 * 1024, cleaner);
- globalLogCleanup(new File(logRootDir), maxSumWorkerLogsSizeMb * 1024 * 1024, cleaner);
- } catch (Exception ex) {
- LOG.error("Exception while cleaning up old log.", ex);
- }
- }
-
- /**
- * Delete the oldest files in each overloaded worker log dir.
- */
- private void perWorkerDirCleanup(File rootDir, long size, DirectoryCleaner cleaner) {
- WorkerLogs.getAllWorkerDirs(rootDir).forEach(Unchecked.consumer(dir -> {
- cleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null);
- }));
- }
-
- /**
- * Delete the oldest files in overloaded worker-artifacts globally.
- */
- private void globalLogCleanup(File rootDir, long size, DirectoryCleaner cleaner) throws Exception {
- List<File> workerDirs = new ArrayList<>(WorkerLogs.getAllWorkerDirs(rootDir));
- Set<String> aliveWorkerDirs = new HashSet<>(getAliveWorkerDirs(rootDir));
-
- cleaner.deleteOldestWhileTooLarge(workerDirs, size, false, aliveWorkerDirs);
- }
-
- /**
- * Return a sorted set of java.io.Files that were written by workers that are now active.
- */
- private SortedSet<String> getAliveWorkerDirs(File rootDir) throws Exception {
- Set<String> aliveIds = getAliveIds(Time.currentTimeSecs());
- Set<File> logDirs = WorkerLogs.getAllWorkerDirs(rootDir);
- Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
-
- return idToDir.entrySet().stream()
- .filter(entry -> aliveIds.contains(entry.getKey()))
- .map(Unchecked.function(entry -> entry.getValue().getCanonicalPath()))
- .collect(toCollection(TreeSet::new));
- }
-
- private Map<String, File> identifyWorkerLogDirs(Set<File> logDirs) {
- return logDirs.stream().map(Unchecked.function(logDir -> {
- Optional<File> metaFile = getMetadataFileForWorkerLogDir(logDir);
-
- return metaFile.map(Unchecked.function(m -> new Tuple2<>(getWorkerIdFromMetadataFile(m.getCanonicalPath()), logDir)))
- .orElse(new Tuple2<>("", logDir));
- })).collect(toMap(Tuple2::v1, Tuple2::v2));
- }
-
- private Optional<File> getMetadataFileForWorkerLogDir(File logDir) throws IOException {
- File metaFile = new File(logDir, WORKER_YAML);
- if (metaFile.exists()) {
- return Optional.of(metaFile);
- } else {
- LOG.warn("Could not find {} to clean up for {}", metaFile.getCanonicalPath(), logDir);
- return Optional.empty();
- }
- }
-
- private String getWorkerIdFromMetadataFile(String metaFile) {
- Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(metaFile);
- return ObjectReader.getString(map.get("worker-id"), null);
- }
-
- private Set<String> getAliveIds(int nowSecs) throws Exception {
- return SupervisorUtils.readWorkerHeartbeats(stormConf).entrySet().stream()
- .filter(entry -> Objects.nonNull(entry.getValue())
- && !SupervisorUtils.isWorkerHbTimedOut(nowSecs, entry.getValue(), stormConf))
- .map(Map.Entry::getKey)
- .collect(toCollection(TreeSet::new));
- }
-
- /**
- * Delete the topo dir if it contains zero port dirs.
- */
- private void cleanupEmptyTopoDirectory(File dir) throws IOException {
- File topoDir = dir.getParentFile();
- if (topoDir.listFiles().length == 0) {
- Utils.forceDelete(topoDir.getCanonicalPath());
- }
- }
-
- /**
- * Return a sorted set of java.io.Files that were written by workers that are now dead.
- */
- private SortedSet<File> getDeadWorkerDirs(int nowSecs, Set<File> logDirs) throws Exception {
- if (logDirs.isEmpty()) {
- return new TreeSet<>();
- } else {
- Set<String> aliveIds = getAliveIds(nowSecs);
- Map<String, File> idToDir = identifyWorkerLogDirs(logDirs);
-
- return idToDir.entrySet().stream()
- .filter(entry -> !aliveIds.contains(entry.getKey()))
- .map(Map.Entry::getValue)
- .collect(toCollection(TreeSet::new));
- }
- }
-
- private Set<File> selectDirsForCleanup(int nowMillis, String rootDir) {
- FileFilter fileFilter = mkFileFilterForLogCleanup(nowMillis);
-
- return Arrays.stream(new File(rootDir).listFiles())
- .flatMap(topoDir -> Arrays.stream(topoDir.listFiles(fileFilter)))
- .collect(toCollection(TreeSet::new));
- }
-
- private FileFilter mkFileFilterForLogCleanup(int nowMillis) {
- final int cutoffAgeMillis = cleanupCutoffAgeMillis(nowMillis);
- return file -> !file.isFile() && lastModifiedTimeWorkerLogdir(file) <= cutoffAgeMillis;
- }
-
- /**
- * Return the last modified time for all log files in a worker's log dir.
- * Using stream rather than File.listFiles is to avoid large mem usage
- * when a directory has too many files.
- */
- private long lastModifiedTimeWorkerLogdir(File logDir) {
- Optional<DirectoryStream<Path>> dirStreamOptional = getStreamForDir(logDir);
- long dirModified = logDir.lastModified();
-
- if (!dirStreamOptional.isPresent()) {
- return dirModified;
- }
-
- DirectoryStream<Path> dirStream = dirStreamOptional.get();
- try {
- return StreamSupport.stream(dirStream.spliterator(), false)
- .reduce(dirModified, (maximum, path) -> {
- long curr = path.toFile().lastModified();
- return curr > maximum ? curr : maximum;
- }, BinaryOperator.maxBy(Long::compareTo));
- } catch (Exception ex) {
- LOG.error(ex.getMessage(), ex);
- return dirModified;
- } finally {
- if (DirectoryStream.class.isInstance(dirStream)) {
- IOUtils.closeQuietly(dirStream);
- }
- }
- }
-
- private Optional<DirectoryStream<Path>> getStreamForDir(File file) {
- try {
- return Optional.of(Files.newDirectoryStream(file.toPath()));
- } catch (Exception ex) {
- LOG.error(ex.getMessage(), ex);
- return Optional.empty();
- }
- }
-
- private int cleanupCutoffAgeMillis(int nowMillis) {
- return nowMillis - (ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_AGE_MINS)));
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java
deleted file mode 100644
index dcb2e2c..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogFileDownloader.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.utils;
-
-import javax.ws.rs.core.Response;
-import java.io.File;
-import java.io.IOException;
-
-public class LogFileDownloader {
-
- private final String logRoot;
- private final String daemonLogRoot;
- private final ResourceAuthorizer resourceAuthorizer;
-
- public LogFileDownloader(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
- this.logRoot = logRoot;
- this.daemonLogRoot = daemonLogRoot;
- this.resourceAuthorizer = resourceAuthorizer;
- }
-
- public Response downloadFile(String fileName, String user, boolean isDaemon) throws IOException {
- String rootDir = isDaemon ? daemonLogRoot : logRoot;
- File file = new File(rootDir, fileName).getCanonicalFile();
- if (file.exists()) {
- if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
- return LogviewerResponseBuilder.buildDownloadFile(file);
- } else {
- return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
- }
- } else {
- return LogviewerResponseBuilder.buildResponsePageNotFound();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java
deleted file mode 100644
index bc60e09..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/LogviewerResponseBuilder.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.utils;
-
-import com.google.common.io.ByteStreams;
-import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.ui.UIHelpers;
-
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.Map;
-
-import static j2html.TagCreator.body;
-import static j2html.TagCreator.h2;
-import static javax.ws.rs.core.Response.Status.OK;
-import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
-
-public class LogviewerResponseBuilder {
-
- private LogviewerResponseBuilder() {
- }
-
- public static Response buildSuccessHtmlResponse(String content) {
- return Response.status(OK).entity(content)
- .type(MediaType.TEXT_HTML_TYPE).build();
- }
-
- public static Response buildSuccessJsonResponse(Object entity, String callback, String origin) {
- return new JsonResponseBuilder().setData(entity).setCallback(callback)
- .setHeaders(LogviewerResponseBuilder.getHeadersForSuccessResponse(origin)).build();
- }
-
- public static Response buildDownloadFile(File file) throws IOException {
- // do not close this InputStream in method: it will be used from jetty server
- InputStream is = new FileInputStream(file);
- return Response.status(OK)
- .entity(wrapWithStreamingOutput(is))
- .type(MediaType.APPLICATION_OCTET_STREAM_TYPE)
- .header("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"")
- .build();
- }
-
- public static Response buildResponseUnautohrizedUser(String user) {
- String entity = buildUnauthorizedUserHtml(user);
- return Response.status(OK)
- .entity(entity)
- .type(MediaType.TEXT_HTML_TYPE)
- .build();
- }
-
- public static Response buildResponsePageNotFound() {
- return Response.status(404)
- .entity("Page not found")
- .type(MediaType.TEXT_HTML_TYPE)
- .build();
- }
-
- public static Response buildUnauthorizedUserJsonResponse(String user, String callback) {
- return new JsonResponseBuilder().setData(UIHelpers.unauthorizedUserJson(user))
- .setCallback(callback).setStatus(401).build();
- }
-
- public static Response buildExceptionJsonResponse(Exception ex, String callback) {
- return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(ex))
- .setCallback(callback).setStatus(500).build();
- }
-
- private static Map<String, Object> getHeadersForSuccessResponse(String origin) {
- Map<String, Object> headers = new HashMap<>();
- headers.put("Access-Control-Allow-Origin", origin);
- headers.put("Access-Control-Allow-Credentials", "true");
- return headers;
- }
-
- private static String buildUnauthorizedUserHtml(String user) {
- String content = "User '" + escapeHtml(user) + "' is not authorized.";
- return body(h2(content)).render();
- }
-
- private static StreamingOutput wrapWithStreamingOutput(final InputStream inputStream) {
- return os -> {
- OutputStream wrappedOutputStream = os;
- if (!(os instanceof BufferedOutputStream)) {
- wrappedOutputStream = new BufferedOutputStream(os);
- }
-
- ByteStreams.copy(inputStream, wrappedOutputStream);
-
- wrappedOutputStream.flush();
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java
deleted file mode 100644
index c7a45e7..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/ResourceAuthorizer.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.utils;
-
-import com.google.common.collect.Sets;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.Config;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.security.auth.AuthUtils;
-import org.apache.storm.security.auth.IGroupMappingServiceProvider;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ServerConfigUtils;
-import org.apache.storm.utils.Utils;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.storm.DaemonConfig.UI_FILTER;
-
-public class ResourceAuthorizer {
-
- private final Map<String, Object> stormConf;
- private final IGroupMappingServiceProvider groupMappingServiceProvider;
-
- public ResourceAuthorizer(Map<String, Object> stormConf) {
- this.stormConf = stormConf;
- this.groupMappingServiceProvider = AuthUtils.GetGroupMappingServiceProviderPlugin(stormConf);
- }
-
- public boolean isUserAllowedToAccessFile(String fileName, String user) {
- return isUiFilterNotSet() || isAuthorizedLogUser(user, fileName);
- }
-
- private boolean isUiFilterNotSet() {
- return StringUtils.isBlank(ObjectReader.getString(stormConf.get(UI_FILTER), null));
- }
-
- private boolean isAuthorizedLogUser(String user, String fileName) {
- if (StringUtils.isEmpty(user) || StringUtils.isEmpty(fileName)
- || getLogUserGroupWhitelist(fileName) == null) {
- return false;
- } else {
- Set<String> groups = getUserGroups(user);
- LogUserGroupWhitelist whitelist = getLogUserGroupWhitelist(fileName);
-
- List<String> logsUsers = new ArrayList<>();
- logsUsers.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_USERS)));
- logsUsers.addAll(ObjectReader.getStrings(stormConf.get(Config.NIMBUS_ADMINS)));
- logsUsers.addAll(whitelist.getUserWhitelist());
-
- List<String> logsGroups = new ArrayList<>();
- logsGroups.addAll(ObjectReader.getStrings(stormConf.get(DaemonConfig.LOGS_GROUPS)));
- logsGroups.addAll(whitelist.getGroupWhitelist());
-
- return logsUsers.stream().anyMatch(u -> u.equals(user)) ||
- Sets.intersection(groups, new HashSet<>(logsGroups)).size() < 0;
-
- }
- }
-
- public LogUserGroupWhitelist getLogUserGroupWhitelist(String fileName) {
- File wlFile = ServerConfigUtils.getLogMetaDataFile(fileName);
- Map<String, Object> map = (Map<String, Object>) Utils.readYamlFile(wlFile.getAbsolutePath());
-
- if (map == null) {
- return null;
- }
-
- List<String> logsUsers = ObjectReader.getStrings(map.get(DaemonConfig.LOGS_USERS));
- List<String> logsGroups = ObjectReader.getStrings(map.get(DaemonConfig.LOGS_GROUPS));
- return new LogUserGroupWhitelist(
- logsUsers.isEmpty() ? new HashSet<>() : new HashSet<>(logsUsers),
- logsGroups.isEmpty() ? new HashSet<>() : new HashSet<>(logsGroups)
- );
- }
-
- private Set<String> getUserGroups(String user) {
- try {
- if (StringUtils.isEmpty(user)) {
- return new HashSet<>();
- } else {
- return groupMappingServiceProvider.getGroups(user);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static class LogUserGroupWhitelist {
-
- private Set<String> userWhitelist;
- private Set<String> groupWhitelist;
-
- public LogUserGroupWhitelist(Set<String> userWhitelist, Set<String> groupWhitelist) {
- this.userWhitelist = userWhitelist;
- this.groupWhitelist = groupWhitelist;
- }
-
- public Set<String> getUserWhitelist() {
- return userWhitelist;
- }
-
- public Set<String> getGroupWhitelist() {
- return groupWhitelist;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java
deleted file mode 100644
index c49a4a1..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/utils/WorkerLogs.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.apache.storm.daemon.wip.logviewer.utils;
-
-import org.apache.storm.daemon.DirectoryCleaner;
-import org.apache.storm.utils.Utils;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.stream.Stream;
-
-import static java.util.stream.Collectors.toCollection;
-import static org.apache.storm.daemon.utils.ListFunctionalSupport.takeLast;
-
-public class WorkerLogs {
-
- private WorkerLogs() {
- }
-
- /**
- * Return the path of the worker log with the format of topoId/port/worker.log.*
- */
- public static String getTopologyPortWorkerLog(File file) {
- try {
- String[] splitted = file.getCanonicalPath().split(Utils.FILE_PATH_SEPARATOR);
- List<String> split = takeLast(Arrays.asList(splitted), 3);
-
- return String.join(Utils.FILE_PATH_SEPARATOR, split);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public static List<File> getAllLogsForRootDir(File logDir) throws IOException {
- List<File> files = new ArrayList<>();
- Set<File> topoDirFiles = getAllWorkerDirs(logDir);
- if (topoDirFiles != null) {
- for (File portDir : topoDirFiles) {
- files.addAll(DirectoryCleaner.getFilesForDir(portDir));
- }
- }
-
- return files;
- }
-
- public static Set<File> getAllWorkerDirs(File rootDir) {
- File[] rootDirFiles = rootDir.listFiles();
- if (rootDirFiles != null) {
- return Arrays.stream(rootDirFiles).flatMap(topoDir -> {
- File[] topoFiles = topoDir.listFiles();
- return topoFiles != null ? Arrays.stream(topoFiles) : Stream.empty();
- }).collect(toCollection(TreeSet::new));
- }
-
- return new TreeSet<>();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java
deleted file mode 100644
index c1b4bc7..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerApplication.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.webapp;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.core.Appender;
-import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.appender.RollingFileAppender;
-import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogDownloadHandler;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogPageHandler;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogSearchHandler;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerProfileHandler;
-import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
-import org.apache.storm.security.auth.AuthUtils;
-import org.apache.storm.security.auth.IHttpCredentialsPlugin;
-import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ObjectReader;
-
-import javax.ws.rs.ApplicationPath;
-import javax.ws.rs.core.Application;
-import java.io.File;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.storm.DaemonConfig.LOGVIEWER_APPENDER_NAME;
-
-@ApplicationPath("")
-public class LogviewerApplication extends Application {
- private static Map<String, Object> stormConf;
- private final Set<Object> singletons = new HashSet<Object>();
-
- /**
- * Constructor.
- */
- public LogviewerApplication() {
- String logRoot = ConfigUtils.workerArtifactsRoot(stormConf);
- String daemonLogRoot = logRootDir(ObjectReader.getString(stormConf.get(LOGVIEWER_APPENDER_NAME)));
-
- ResourceAuthorizer resourceAuthorizer = new ResourceAuthorizer(stormConf);
-
- LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, resourceAuthorizer);
- LogviewerProfileHandler profileHandler = new LogviewerProfileHandler(logRoot, resourceAuthorizer);
- LogviewerLogDownloadHandler logDownloadHandler = new LogviewerLogDownloadHandler(logRoot, daemonLogRoot,
- resourceAuthorizer);
- LogviewerLogSearchHandler logSearchHandler = new LogviewerLogSearchHandler(stormConf, logRoot, daemonLogRoot,
- resourceAuthorizer);
- IHttpCredentialsPlugin httpCredsHandler = AuthUtils.GetUiHttpCredentialsPlugin(stormConf);
-
- singletons.add(new LogviewerResource(logviewer, profileHandler, logDownloadHandler, logSearchHandler, httpCredsHandler));
- singletons.add(new AuthorizationExceptionMapper());
- }
-
- @Override
- public Set<Object> getSingletons() {
- return singletons;
- }
-
- public static void setup(Map<String, Object> stormConf) {
- LogviewerApplication.stormConf = stormConf;
- }
-
- /**
- * Given an appender name, as configured, get the parent directory of the appender's log file.
- * Note that if anything goes wrong, this will throw an Error and exit.
- */
- private String logRootDir(String appenderName) {
- Appender appender = ((LoggerContext) LogManager.getContext()).getConfiguration().getAppender(appenderName);
- if (appenderName != null && appender != null && RollingFileAppender.class.isInstance(appender)) {
- return new File(((RollingFileAppender) appender).getFileName()).getParent();
- } else {
- throw new RuntimeException("Log viewer could not find configured appender, or the appender is not a FileAppender. "
- + "Please check that the appender name configured in storm and log4j agree.");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java
deleted file mode 100644
index fd8ea97..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/webapp/LogviewerResource.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.wip.logviewer.webapp;
-
-import com.codahale.metrics.Meter;
-import org.apache.commons.lang.BooleanUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.daemon.common.JsonResponseBuilder;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogDownloadHandler;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogPageHandler;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerLogSearchHandler;
-import org.apache.storm.daemon.wip.logviewer.handler.LogviewerProfileHandler;
-import org.apache.storm.metric.StormMetricsRegistry;
-import org.apache.storm.security.auth.IHttpCredentialsPlugin;
-import org.apache.storm.ui.InvalidRequestException;
-import org.apache.storm.ui.UIHelpers;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.net.URLDecoder;
-import java.util.Map;
-
-@Path("/")
-public class LogviewerResource {
- private static final Logger LOG = LoggerFactory.getLogger(LogviewerResource.class);
-
- private static final Meter meterLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-log-page-http-requests");
- private static final Meter meterDaemonLogPageHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-daemonlog-page-http-requests");
- private static final Meter meterDownloadLogFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-file-http-requests");
- private static final Meter meterDownloadLogDaemonFileHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-download-log-daemon-file-http-requests");
- private static final Meter meterListLogsHttpRequests = StormMetricsRegistry.registerMeter("logviewer:num-list-logs-http-requests");
-
- private final LogviewerLogPageHandler logviewer;
- private final LogviewerProfileHandler profileHandler;
- private final LogviewerLogDownloadHandler logDownloadHandler;
- private final LogviewerLogSearchHandler logSearchHandler;
- private final IHttpCredentialsPlugin httpCredsHandler;
-
- public LogviewerResource(LogviewerLogPageHandler logviewerParam, LogviewerProfileHandler profileHandler,
- LogviewerLogDownloadHandler logDownloadHandler, LogviewerLogSearchHandler logSearchHandler,
- IHttpCredentialsPlugin httpCredsHandler) {
- this.logviewer = logviewerParam;
- this.profileHandler = profileHandler;
- this.logDownloadHandler = logDownloadHandler;
- this.logSearchHandler = logSearchHandler;
- this.httpCredsHandler = httpCredsHandler;
- }
-
- @GET
- @Path("/log")
- public Response log(@Context HttpServletRequest request) throws IOException {
- meterLogPageHttpRequests.mark();
-
- try {
- String user = httpCredsHandler.getUserName(request);
- Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null;
- Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null;
- String decodedFileName = URLDecoder.decode(request.getParameter("file"));
- String grep = request.getParameter("grep");
- return logviewer.logPage(decodedFileName, start, length, grep, user);
- } catch (InvalidRequestException e) {
- LOG.error(e.getMessage(), e);
- return Response.status(400).entity(e.getMessage()).build();
- }
- }
-
- @GET
- @Path("/daemonlog")
- public Response daemonLog(@Context HttpServletRequest request) throws IOException {
- meterDaemonLogPageHttpRequests.mark();
-
- try {
- String user = httpCredsHandler.getUserName(request);
- Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null;
- Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null;
- String decodedFileName = URLDecoder.decode(request.getParameter("file"));
- String grep = request.getParameter("grep");
- return logviewer.daemonLogPage(decodedFileName, start, length, grep, user);
- } catch (InvalidRequestException e) {
- LOG.error(e.getMessage(), e);
- return Response.status(400).entity(e.getMessage()).build();
- }
- }
-
- @GET
- @Path("/searchLogs")
- public Response searchLogs(@Context HttpServletRequest request) throws IOException {
- String user = httpCredsHandler.getUserName(request);
- String topologyId = request.getParameter("topoId");
- String portStr = request.getParameter("port");
- String callback = request.getParameter("callback");
- String origin = request.getHeader("Origin");
-
- return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
- }
-
- @GET
- @Path("/listLogs")
- public Response listLogs(@Context HttpServletRequest request) throws IOException {
- meterListLogsHttpRequests.mark();
-
- String user = httpCredsHandler.getUserName(request);
- String topologyId = request.getParameter("topoId");
- String portStr = request.getParameter("port");
- String callback = request.getParameter("callback");
- String origin = request.getHeader("Origin");
-
- return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin);
- }
-
- @GET
- @Path("/dumps/{topo-id}/{host-port}")
- public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
- @Context HttpServletRequest request) throws IOException {
- String user = httpCredsHandler.getUserName(request);
- return profileHandler.listDumpFiles(topologyId, hostPort, user);
- }
-
- @GET
- @Path("/dumps/{topo-id}/{host-port}/{filename}")
- public Response downloadDumpFile(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort,
- @PathParam("filename") String fileName, @Context HttpServletRequest request) throws IOException {
- String user = httpCredsHandler.getUserName(request);
- return profileHandler.downloadDumpFile(topologyId, hostPort, fileName, user);
- }
-
- @GET
- @Path("/download")
- public Response downloadLogFile(@Context HttpServletRequest request) throws IOException {
- meterDownloadLogFileHttpRequests.mark();
-
- String user = httpCredsHandler.getUserName(request);
- String file = request.getParameter("file");
- String decodedFileName = URLDecoder.decode(file);
- return logDownloadHandler.downloadLogFile(decodedFileName, user);
- }
-
- @GET
- @Path("/daemondownload")
- public Response downloadDaemonLogFile(@Context HttpServletRequest request) throws IOException {
- meterDownloadLogDaemonFileHttpRequests.mark();
-
- String user = httpCredsHandler.getUserName(request);
- String file = request.getParameter("file");
- String decodedFileName = URLDecoder.decode(file);
- return logDownloadHandler.downloadDaemonLogFile(decodedFileName, user);
- }
-
- @GET
- @Path("/search/{file}")
- public Response search(@PathParam("file") String file, @Context HttpServletRequest request) throws IOException {
- String user = httpCredsHandler.getUserName(request);
- boolean isDaemon = StringUtils.equals(request.getParameter("is-daemon"), "yes");
- String decodedFileName = URLDecoder.decode(file);
- String searchString = request.getParameter("search-string");
- String numMatchesStr = request.getParameter("num-matches");
- String startByteOffset = request.getParameter("start-byte-offset");
- String callback = request.getParameter("callback");
- String origin = request.getHeader("Origin");
-
- try {
- return logSearchHandler.searchLogFile(decodedFileName, user, isDaemon, searchString, numMatchesStr,
- startByteOffset, callback, origin);
- } catch (InvalidRequestException e) {
- LOG.error(e.getMessage(), e);
- return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(e)).setCallback(callback)
- .setStatus(400).build();
- }
- }
-
- @GET
- @Path("/deepSearch/{topoId}")
- public Response deepSearch(@PathParam("topoId") String topologyId,
- @Context HttpServletRequest request) throws IOException {
- String user = httpCredsHandler.getUserName(request);
- String searchString = request.getParameter("search-string");
- String numMatchesStr = request.getParameter("num-matches");
- String portStr = request.getParameter("port");
- String startFileOffset = request.getParameter("start-file-offset");
- String startByteOffset = request.getParameter("start-byte-offset");
- String searchArchived = request.getParameter("search-archived");
- String callback = request.getParameter("callback");
- String origin = request.getHeader("Origin");
-
- return logSearchHandler.deepSearchLogsForTopology(topologyId, user, searchString, numMatchesStr, portStr,
- startFileOffset, startByteOffset, BooleanUtils.toBooleanObject(searchArchived), callback, origin);
- }
-
- private int parseIntegerFromMap(Map map, String parameterKey) throws InvalidRequestException {
- try {
- return Integer.parseInt(((String[]) map.get(parameterKey))[0]);
- } catch (NumberFormatException ex) {
- throw new InvalidRequestException("Could not make an integer out of the query parameter '"
- + parameterKey + "'", ex);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java
new file mode 100644
index 0000000..6abe153
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/LogviewerTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class LogviewerTest {
+ private Path mkMockPath(File file) {
+ Path mockPath = mock(Path.class);
+ when(mockPath.toFile()).thenReturn(file);
+ return mockPath;
+ }
+
+ private DirectoryStream<Path> mkDirectoryStream(List<Path> listOfPaths) {
+ return new DirectoryStream<Path>() {
+ @Override
+ public Iterator<Path> iterator() {
+ return listOfPaths.iterator();
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/44b268ba/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
new file mode 100644
index 0000000..4b76abd
--- /dev/null
+++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.logviewer.handler;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.utils.Utils;
+import org.assertj.core.util.Lists;
+import org.junit.Test;
+
+import javax.ws.rs.core.Response;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.FileAttribute;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class LogviewerLogPageHandlerTest {
+
+ /**
+ * list-log-files filter selects the correct log files to return.
+ */
+ @Test
+ public void testListLogFiles() throws IOException {
+ FileAttribute[] attrs = new FileAttribute[0];
+ String rootPath = Files.createTempDirectory("workers-artifacts", attrs).toFile().getCanonicalPath();
+ File file1 = new File(String.join(File.separator, rootPath, "topoA", "1111"), "worker.log");
+ File file2 = new File(String.join(File.separator, rootPath, "topoA", "2222"), "worker.log");
+ File file3 = new File(String.join(File.separator, rootPath, "topoB", "1111"), "worker.log");
+
+ file1.getParentFile().mkdirs();
+ file2.getParentFile().mkdirs();
+ file3.getParentFile().mkdirs();
+ file1.createNewFile();
+ file2.createNewFile();
+ file3.createNewFile();
+
+ String origin = "www.origin.server.net";
+
+ LogviewerLogPageHandler handler = new LogviewerLogPageHandler(rootPath, null,
+ new ResourceAuthorizer(Utils.readStormConfig()));
+
+ Response expectedAll = LogviewerResponseBuilder.buildSuccessJsonResponse(
+ Lists.newArrayList("topoA/port1/worker.log", "topoA/port2/worker.log", "topoB/port1/worker.log"),
+ null,
+ origin
+ );
+
+ Response expectedFilterPort = LogviewerResponseBuilder.buildSuccessJsonResponse(
+ Lists.newArrayList("topoA/port1/worker.log", "topoB/port1/worker.log"),
+ null,
+ origin
+ );
+
+ Response expectedFilterTopoId = LogviewerResponseBuilder.buildSuccessJsonResponse(
+ Lists.newArrayList("topoB/port1/worker.log"),
+ null,
+ origin
+ );
+
+ Response returnedAll = handler.listLogFiles("user", null, null, null, origin);
+ Response returnedFilterPort = handler.listLogFiles("user", 1111, null, null, origin);
+ Response returnedFilterTopoId = handler.listLogFiles("user", null, "topoB", null, origin);
+
+ Utils.forceDelete(rootPath);
+
+ assertEqualsJsonResponse(expectedAll, returnedAll, List.class);
+ assertEqualsJsonResponse(expectedFilterPort, returnedFilterPort, List.class);
+ assertEqualsJsonResponse(expectedFilterTopoId, returnedFilterTopoId, List.class);
+ }
+
+ private <T> void assertEqualsJsonResponse(Response expected, Response actual, Class<T> entityClass) throws IOException {
+ ObjectMapper objectMapper = new ObjectMapper();
+ T entityFromExpected = objectMapper.readValue((String) expected.getEntity(), entityClass);
+ T actualFromExpected = objectMapper.readValue((String) expected.getEntity(), entityClass);
+ assertEquals(entityFromExpected, actualFromExpected);
+
+ assertEquals(expected.getStatus(), actual.getStatus());
+ assertTrue(expected.getHeaders().equalsIgnoreValueOrder(actual.getHeaders()));
+ }
+}
[17/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* fix a bug: parameter swapped (address another missing places)
* swap parameter sequence to make it consistent with other method
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8b4a1ce3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8b4a1ce3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8b4a1ce3
Branch: refs/heads/master
Commit: 8b4a1ce3a13bc1c51297cc80625c51e55c7790a0
Parents: 35ca265
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Jul 24 13:38:07 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Jul 24 13:38:07 2017 +0900
----------------------------------------------------------------------
.../storm/daemon/logviewer/handler/LogviewerLogPageHandler.java | 2 +-
.../storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java | 2 +-
.../org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java | 2 +-
.../apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8b4a1ce3/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
index c6e9828..5c883d9 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java
@@ -176,7 +176,7 @@ public class LogviewerLogPageHandler {
public Response logPage(String fileName, Integer start, Integer length, String grep, String user)
throws IOException, InvalidRequestException {
String rootDir = logRoot;
- if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ if (resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
workerLogs.setLogFilePermission(fileName);
File file = new File(rootDir, fileName).getCanonicalFile();
http://git-wip-us.apache.org/repos/asf/storm/blob/8b4a1ce3/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
index 117cb86..76f1683 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java
@@ -119,7 +119,7 @@ public class LogviewerLogSearchHandler {
File file = new File(rootDir, fileName).getCanonicalFile();
Response response;
if (file.exists()) {
- if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
Integer numMatchesInt = numMatchesStr != null ? tryParseIntParam("num-matches", numMatchesStr) : null;
Integer offsetInt = offsetStr != null ? tryParseIntParam("start-byte-offset", offsetStr) : null;
http://git-wip-us.apache.org/repos/asf/storm/blob/8b4a1ce3/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
index 62c08da..3984f03 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java
@@ -54,7 +54,7 @@ public class LogFileDownloader {
String rootDir = isDaemon ? daemonLogRoot : logRoot;
File file = new File(rootDir, fileName).getCanonicalFile();
if (file.exists()) {
- if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ if (isDaemon || resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
return LogviewerResponseBuilder.buildDownloadFile(file);
} else {
return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
http://git-wip-us.apache.org/repos/asf/storm/blob/8b4a1ce3/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
index 16f408a..286c176 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java
@@ -61,7 +61,7 @@ public class ResourceAuthorizer {
* @param fileName file name to access
* @param user username
*/
- public boolean isUserAllowedToAccessFile(String fileName, String user) {
+ public boolean isUserAllowedToAccessFile(String user, String fileName) {
return isUiFilterNotSet() || isAuthorizedLogUser(user, fileName);
}
[10/19] storm git commit: STORM-1280 port
backtype.storm.daemon.logviewer to java
Posted by bo...@apache.org.
STORM-1280 port backtype.storm.daemon.logviewer to java
* port logviewer to Java using Dropwizard
* follow implementation of DRPC Server
* add '/api/v1/' to the logviewer API endpoints
* seems like it is not possible (or not easy) to serve both API endpoints and static resources in same servlet
* reflect the change to the static resources
* TODO: logviewer test should be ported to Java since ported version of Logviewer is placed to storm-webapp module
* can't remove logviewer.clj for now, hence adding 'wip' to package to avoid clash
* ported version of Logviewer is manually tested, not testing with logviewer-test
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/11a79053
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/11a79053
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/11a79053
Branch: refs/heads/master
Commit: 11a79053f7a9250b7347013cafff0e8eec79ec70
Parents: 559053d
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Jul 11 13:49:56 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Jul 14 12:11:41 2017 +0900
----------------------------------------------------------------------
bin/storm.py | 8 +-
pom.xml | 3 +
.../org/apache/storm/daemon/StormCommon.java | 2 -
storm-core/src/clj/org/apache/storm/ui/core.clj | 10 +-
.../apache/storm/daemon/DirectoryCleaner.java | 52 +-
storm-core/src/ui/public/component.html | 2 +-
.../src/ui/public/deep_search_result.html | 2 +-
storm-core/src/ui/public/logviewer_search.html | 2 +-
storm-core/src/ui/public/search_result.html | 4 +-
.../storm/hack/StormShadeTransformer.java | 5 -
.../reporters/JmxPreparableReporter.java | 1 -
.../storm/daemon/supervisor/BasicContainer.java | 1 -
storm-webapp/pom.xml | 53 +-
.../common/AuthorizationExceptionMapper.java | 40 ++
.../daemon/common/JsonResponseBuilder.java | 66 ++
.../webapp/AuthorizationExceptionMapper.java | 40 --
.../daemon/drpc/webapp/DRPCApplication.java | 1 +
.../daemon/utils/ListFunctionalSupport.java | 69 ++
.../apache/storm/daemon/utils/StreamUtil.java | 38 +
.../apache/storm/daemon/utils/URLBuilder.java | 43 ++
.../daemon/wip/logviewer/LogviewerConstant.java | 23 +
.../daemon/wip/logviewer/LogviewerServer.java | 174 +++++
.../handler/LogviewerLogDownloadHandler.java | 43 ++
.../handler/LogviewerLogPageHandler.java | 412 +++++++++++
.../handler/LogviewerLogSearchHandler.java | 686 +++++++++++++++++++
.../handler/LogviewerProfileHandler.java | 115 ++++
.../daemon/wip/logviewer/utils/LogCleaner.java | 296 ++++++++
.../wip/logviewer/utils/LogFileDownloader.java | 51 ++
.../utils/LogviewerResponseBuilder.java | 118 ++++
.../wip/logviewer/utils/ResourceAuthorizer.java | 129 ++++
.../daemon/wip/logviewer/utils/WorkerLogs.java | 63 ++
.../logviewer/webapp/LogviewerApplication.java | 94 +++
.../wip/logviewer/webapp/LogviewerResource.java | 221 ++++++
33 files changed, 2773 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index 8f7a635..dade6b5 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -808,12 +808,16 @@ def logviewer():
"-DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector",
"-Dlog4j.configurationFile=" + os.path.join(get_log4j2_conf_dir(), "cluster.xml")
]
+
+ allextrajars = get_wildcard_dir(STORM_WEBAPP_LIB_DIR)
+ allextrajars.append(CLUSTER_CONF_DIR)
exec_storm_class(
- "org.apache.storm.daemon.logviewer",
+ "org.apache.storm.daemon.wip.logviewer.LogviewerServer",
jvmtype="-server",
daemonName="logviewer",
jvmopts=jvmopts,
- extrajars=[STORM_DIR, CLUSTER_CONF_DIR])
+ extrajars=allextrajars)
+
def drpcclient(*args):
"""Syntax: [storm drpc-client [options] ([function argument]*)|(argument*)]
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index eb3ba05..ffee0cd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -310,6 +310,9 @@
<qpid.version>0.32</qpid.version>
<azure-eventhubs.version>0.13.1</azure-eventhubs.version>
<jersey.version>2.24.1</jersey.version>
+ <dropwizard.version>1.1.2</dropwizard.version>
+ <j2html.version>1.0.0</j2html.version>
+ <jool.version>0.9.12</jool.version>
<!-- see intellij profile below... This fixes an annoyance with intellij -->
<provided.scope>provided</provided.scope>
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java b/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
index 3f85a13..9b5163b 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java
@@ -52,10 +52,8 @@ import org.apache.storm.task.IBolt;
import org.apache.storm.task.WorkerTopologyContext;
import org.apache.storm.tuple.Fields;
import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ThriftTopologyUtils;
import org.apache.storm.utils.Utils;
import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ThriftTopologyUtils;
import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-core/src/clj/org/apache/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/core.clj b/storm-core/src/clj/org/apache/storm/ui/core.clj
index d1bdee8..5b0613c 100644
--- a/storm-core/src/clj/org/apache/storm/ui/core.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/core.clj
@@ -134,12 +134,12 @@
(defn logviewer-link [host fname secure?]
(if (and secure? (*STORM-CONF* LOGVIEWER-HTTPS-PORT))
- (UIHelpers/urlFormat "https://%s:%s/log?file=%s"
+ (UIHelpers/urlFormat "https://%s:%s/api/v1/log?file=%s"
(to-array
[host
(*STORM-CONF* LOGVIEWER-HTTPS-PORT)
fname]))
- (UIHelpers/urlFormat "http://%s:%s/log?file=%s"
+ (UIHelpers/urlFormat "http://%s:%s/api/v1/log?file=%s"
(to-array
[host
(*STORM-CONF* LOGVIEWER-PORT)
@@ -156,10 +156,10 @@
(logviewer-link host fname secure?))))
(defn nimbus-log-link [host]
- (UIHelpers/urlFormat "http://%s:%s/daemonlog?file=nimbus.log" (to-array [host (*STORM-CONF* LOGVIEWER-PORT)])))
+ (UIHelpers/urlFormat "http://%s:%s/api/v1/daemonlog?file=nimbus.log" (to-array [host (*STORM-CONF* LOGVIEWER-PORT)])))
(defn supervisor-log-link [host]
- (UIHelpers/urlFormat "http://%s:%s/daemonlog?file=supervisor.log" (to-array [host (*STORM-CONF* LOGVIEWER-PORT)])))
+ (UIHelpers/urlFormat "http://%s:%s/api/v1/daemonlog?file=supervisor.log" (to-array [host (*STORM-CONF* LOGVIEWER-PORT)])))
(defn get-error-data
[error]
@@ -185,7 +185,7 @@
(.get_error_time_secs ^ErrorInfo error)))
(defn worker-dump-link [host port topology-id]
- (UIHelpers/urlFormat "http://%s:%s/dumps/%s/%s"
+ (UIHelpers/urlFormat "http://%s:%s/api/v1/dumps/%s/%s"
(to-array [(URLEncoder/encode host)
(*STORM-CONF* LOGVIEWER-PORT)
(URLEncoder/encode topology-id)
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
index 3a6eeba..dc76157 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/DirectoryCleaner.java
@@ -55,15 +55,15 @@ public class DirectoryCleaner {
/**
* If totalSize of files exceeds the either the per-worker quota or global quota,
* Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs.
- * We use the parameter for_per_dir to switch between the two deletion modes.
+ * We use the parameter forPerDir to switch between the two deletion modes.
* @param dirs the list of directories to be scanned for deletion
* @param quota the per-dir quota or the total quota for the all directories
- * @param for_per_dir if true, deletion happens for a single dir; otherwise, for all directories globally
- * @param active_dirs only for global deletion, we want to skip the active logs in active_dirs
+ * @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories globally
+ * @param activeDirs only for global deletion, we want to skip the active logs in activeDirs
* @return number of files deleted
*/
public int deleteOldestWhileTooLarge(List<File> dirs,
- long quota, boolean for_per_dir, Set<String> active_dirs) throws IOException {
+ long quota, boolean forPerDir, Set<String> activeDirs) throws IOException {
final int PQ_SIZE = 1024; // max number of files to delete for every round
final int MAX_ROUNDS = 512; // max rounds of scanning the dirs
long totalSize = 0;
@@ -73,10 +73,17 @@ public class DirectoryCleaner {
try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
for (Path path : stream) {
File file = path.toFile();
+
+ if (isFileEligibleToSkipDelete(forPerDir, activeDirs, dir, file)) {
+ continue; // skip adding length
+ }
+
totalSize += file.length();
}
}
}
+
+ LOG.debug("totalSize: {} quota: {}", totalSize, quota);
long toDeleteSize = totalSize - quota;
if (toDeleteSize <= 0) {
return deletedFiles;
@@ -100,20 +107,8 @@ public class DirectoryCleaner {
try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
for (Path path : stream) {
File file = path.toFile();
- if (for_per_dir) {
- if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
- continue; // skip active log files
- }
- } else { // for global cleanup
- if (active_dirs.contains(dir.getCanonicalPath())) { // for an active worker's dir, make sure for the last "/"
- if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
- continue; // skip active log files
- }
- } else {
- if (META_LOG_PATTERN.matcher(file.getName()).matches()) {
- continue; // skip yaml and pid files
- }
- }
+ if (isFileEligibleToSkipDelete(forPerDir, activeDirs, dir, file)) {
+ continue;
}
if (pq.size() < PQ_SIZE) {
pq.offer(file);
@@ -142,7 +137,7 @@ public class DirectoryCleaner {
pq.clear();
round++;
if (round >= MAX_ROUNDS) {
- if (for_per_dir) {
+ if (forPerDir) {
LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " +
"a single directory : {}, will delete the rest files in next interval.",
MAX_ROUNDS, dirs.get(0).getCanonicalPath());
@@ -156,6 +151,25 @@ public class DirectoryCleaner {
return deletedFiles;
}
+ private boolean isFileEligibleToSkipDelete(boolean forPerDir, Set<String> activeDirs, File dir, File file) throws IOException {
+ if (forPerDir) {
+ if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
+ return true;
+ }
+ } else { // for global cleanup
+ if (activeDirs.contains(dir.getCanonicalPath())) { // for an active worker's dir, make sure for the last "/"
+ if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
+ return true;
+ }
+ } else {
+ if (META_LOG_PATTERN.matcher(file.getName()).matches()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
// Note that to avoid memory problem, we only return the first 1024 files in a directory
public static List<File> getFilesForDir(File dir) throws IOException {
List<File> files = new ArrayList<File>();
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html
index 942b12d..4d602f2 100644
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@ -182,7 +182,7 @@ $(document).ready(function() {
var loc = $(row[3])[0]; // logviewer URL
return '<input type="checkbox" class="workerActionCheckbox"'+
'id="'+checkboxId+'" value="'+host_port+'"'+checkedString+'/> '+
- '<a href="'+loc.protocol+'//'+loc.host+'/dumps/'+topologyId+'/'+
+ '<a href="'+loc.protocol+'//'+loc.host+'/api/v1/dumps/'+topologyId+'/'+
encodeURIComponent(host_port)+'">files</a>';
break;
case 'sort':
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-core/src/ui/public/deep_search_result.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/deep_search_result.html b/storm-core/src/ui/public/deep_search_result.html
index 5de2ab8..5f5e5c6 100644
--- a/storm-core/src/ui/public/deep_search_result.html
+++ b/storm-core/src/ui/public/deep_search_result.html
@@ -120,7 +120,7 @@ $(document).ready(function() {
for (var host in distinct_hosts) {
- var searchURL = "http://"+host+":"+logviewerPort+"/deepSearch/"+id+"?search-string="+search+"&num-matches="+count+"&port="+port;
+ var searchURL = "http://"+host+":"+logviewerPort+"/api/v1/deepSearch/"+id+"?search-string="+search+"&num-matches="+count+"&port="+port;
if(search_archived)
searchURL += "&search-archived=" + search_archived;
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-core/src/ui/public/logviewer_search.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/logviewer_search.html b/storm-core/src/ui/public/logviewer_search.html
index 3a1682e..89a7854 100644
--- a/storm-core/src/ui/public/logviewer_search.html
+++ b/storm-core/src/ui/public/logviewer_search.html
@@ -55,7 +55,7 @@ $(document).ready(function() {
$("#search-form").append(Mustache.render($(template).filter("#search-single-file").html(),{file: file, search: search, isDaemon: isDaemon}));
var result = $("#result");
- var url = "/search/"+encodeURIComponent(file)+"?search-string="+search+"&start-byte-offset="+offset+"&is-daemon="+isDaemon;
+ var url = "/api/v1/search/"+encodeURIComponent(file)+"?search-string="+search+"&start-byte-offset="+offset+"&is-daemon="+isDaemon;
$.getJSON(url,function(response,status,jqXHR) {
response.file = file;
result.append(Mustache.render($(template).filter("#logviewer-search-result-template").html(),response));
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-core/src/ui/public/search_result.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/search_result.html b/storm-core/src/ui/public/search_result.html
index ee0efd8..216a700 100644
--- a/storm-core/src/ui/public/search_result.html
+++ b/storm-core/src/ui/public/search_result.html
@@ -68,9 +68,9 @@ $(document).ready(function() {
var port = response.hostPortList[index].port;
var elemId = response.hostPortList[index].elemId;
var file = id+"/"+port+"/worker.log";
- var searchURL = "http://"+host+":"+logviewerPort+"/search/"+encodeURIComponent(file)+"?search-string="+search+"&num-matches="+count;
+ var searchURL = "http://"+host+":"+logviewerPort+"/api/v1/search/"+encodeURIComponent(file)+"?search-string="+search+"&num-matches="+count;
if (searchArchived != "") {
- searchURL = "http://"+host+":"+logviewerPort+"/deepSearch/"+id+"?search-string="+search+"&num-matches="+count+"&search-archived=true&port="+port;
+ searchURL = "http://"+host+":"+logviewerPort+"/api/v1/deepSearch/"+id+"?search-string="+search+"&num-matches="+count+"&search-archived=true&port="+port;
}
$.ajax({dataType: "json",
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java
----------------------------------------------------------------------
diff --git a/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java b/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java
index ba1b1ec..8afbcea 100644
--- a/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java
+++ b/storm-rename-hack/src/main/java/org/apache/storm/hack/StormShadeTransformer.java
@@ -19,13 +19,8 @@
package org.apache.storm.hack;
import org.apache.storm.daemon.JarTransformer;
-import org.apache.storm.hack.relocation.Relocator;
-import org.apache.storm.hack.relocation.SimpleRelocator;
-import org.apache.storm.hack.resource.ClojureTransformer;
-import org.apache.storm.hack.resource.ResourceTransformer;
import java.io.*;
-import java.util.Arrays;
public class StormShadeTransformer implements JarTransformer {
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
index 48eef02..21aab16 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
@@ -19,7 +19,6 @@ package org.apache.storm.daemon.metrics.reporters;
import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.MetricRegistry;
-import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.metrics.MetricsUtils;
import org.apache.storm.utils.ObjectReader;
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
index f5eed43..fe088dc 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -22,7 +22,6 @@ import static org.apache.storm.utils.Utils.OR;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/storm-webapp/pom.xml b/storm-webapp/pom.xml
index eed6050..eeb6ee6 100644
--- a/storm-webapp/pom.xml
+++ b/storm-webapp/pom.xml
@@ -35,7 +35,7 @@
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>2.0.0-SNAPSHOT</version>
- <scope>provided</scope>
+ <scope>${provided.scope}</scope>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
@@ -75,24 +75,49 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+
<dependency>
- <groupId>org.glassfish.jersey.core</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
+ <groupId>io.dropwizard</groupId>
+ <artifactId>dropwizard-core</artifactId>
+ <version>${dropwizard.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-servlet-core</artifactId>
- </dependency>
+ <groupId>io.dropwizard</groupId>
+ <artifactId>dropwizard-assets</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.j2html</groupId>
+ <artifactId>j2html</artifactId>
+ <version>${j2html.version}</version>
+ </dependency>
<dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-jetty-http</artifactId>
+ <groupId>org.jooq</groupId>
+ <artifactId>jool</artifactId>
+ <version>${jool.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard</groupId>
+ <artifactId>dropwizard-testing</artifactId>
+ <version>${dropwizard.version}</version>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
@@ -118,7 +143,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>4</maxAllowedViolations>
+ <maxAllowedViolations>500</maxAllowedViolations>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java
new file mode 100644
index 0000000..57467e1
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.common;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.storm.generated.AuthorizationException;
+import org.json.simple.JSONValue;
+
+@Provider
+public class AuthorizationExceptionMapper implements ExceptionMapper<AuthorizationException> {
+ @Override
+ public Response toResponse(AuthorizationException ex) {
+ Map<String, String> body = new HashMap<>();
+ body.put("error", "Not Authorized");
+ body.put("errorMessage", ex.get_msg());
+ return Response.status(403).entity(JSONValue.toJSONString(body)).type("application/json").build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
new file mode 100644
index 0000000..2048d81
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.common;
+
+import org.apache.storm.ui.UIHelpers;
+
+import javax.ws.rs.core.Response;
+import java.util.Collections;
+import java.util.Map;
+
+public class JsonResponseBuilder {
+ private Object data;
+ private String callback;
+ private boolean needSerialize = true;
+ private int status = 200;
+ private Map<String, Object> headers = Collections.emptyMap();
+
+ public JsonResponseBuilder setData(Object data) {
+ this.data = data;
+ return this;
+ }
+
+ public JsonResponseBuilder setCallback(String callback) {
+ this.callback = callback;
+ return this;
+ }
+
+ public JsonResponseBuilder setNeedSerialize(boolean needSerialize) {
+ this.needSerialize = needSerialize;
+ return this;
+ }
+
+ public JsonResponseBuilder setStatus(int status) {
+ this.status = status;
+ return this;
+ }
+
+ public JsonResponseBuilder setHeaders(Map<String, Object> headers) {
+ this.headers = headers;
+ return this;
+ }
+
+ public Response build() {
+ String body = UIHelpers.getJsonResponseBody(data, callback, needSerialize);
+ Map<String, Object> respHeaders = UIHelpers.getJsonResponseHeaders(callback, headers);
+ Response.ResponseBuilder respBuilder = Response.status(status).entity(body);
+ respHeaders.forEach(respBuilder::header);
+ return respBuilder.build();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/AuthorizationExceptionMapper.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/AuthorizationExceptionMapper.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/AuthorizationExceptionMapper.java
deleted file mode 100644
index fcef715..0000000
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/AuthorizationExceptionMapper.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.daemon.drpc.webapp;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.ExceptionMapper;
-import javax.ws.rs.ext.Provider;
-
-import org.apache.storm.generated.AuthorizationException;
-import org.json.simple.JSONValue;
-
-@Provider
-public class AuthorizationExceptionMapper implements ExceptionMapper<AuthorizationException> {
- @Override
- public Response toResponse(AuthorizationException ex) {
- Map<String, String> body = new HashMap<>();
- body.put("error", "Not Authorized");
- body.put("errorMessage", ex.get_msg());
- return Response.status(403).entity(JSONValue.toJSONString(body)).type("application/json").build();
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
index 0074b05..db42310 100644
--- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java
@@ -25,6 +25,7 @@ import javax.ws.rs.ApplicationPath;
import javax.ws.rs.core.Application;
import org.apache.storm.daemon.drpc.DRPC;
+import org.apache.storm.daemon.common.AuthorizationExceptionMapper;
@ApplicationPath("")
public class DRPCApplication extends Application {
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java
new file mode 100644
index 0000000..83b3588
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.utils;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class ListFunctionalSupport {
+ public static <T> T first(List<T> list) {
+ if (list == null || list.size() <= 0) {
+ return null;
+ }
+
+ return list.get(0);
+ }
+
+ public static <T> List<T> takeLast(List<T> list, int count) {
+ if (list == null) {
+ return null;
+ }
+
+ if (list.size() <= count) {
+ return list;
+ } else {
+ return list.stream()
+ .skip(list.size() - count)
+ .limit(count)
+ .collect(Collectors.toList());
+ }
+ }
+
+ public static <T> List<T> drop(List<T> list, int count) {
+ if (list == null) {
+ return null;
+ }
+
+ return list.stream()
+ .skip(count)
+ .collect(Collectors.toList());
+ }
+
+ public static <T> List<T> rest(List<T> list) {
+ return drop(list, 1);
+ }
+
+ public static <T> T last(List<T> list) {
+ if (list == null || list.size() <= 0) {
+ return null;
+ }
+
+ return list.get(list.size() - 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java
new file mode 100644
index 0000000..24c630d
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class StreamUtil {
+ private StreamUtil() {
+ }
+
+ /**
+ * FileInputStream#skip may not work the first time, so ensure it successfully skips the given number of bytes.
+ */
+ public static void skipBytes(InputStream stream, Integer n) throws IOException {
+ long skipped = 0;
+ do {
+ skipped = skipped + stream.skip(n - skipped);
+ } while (skipped < n);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java
new file mode 100644
index 0000000..d30ac65
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/URLBuilder.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.utils;
+
+import java.net.URLEncoder;
+import java.util.Map;
+
+import static java.util.stream.Collectors.joining;
+
+public class URLBuilder {
+ private URLBuilder() {
+ }
+
+ public static String build(String urlPath, Map<String, Object> parameters) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(urlPath);
+ if (parameters.size() > 0) {
+ sb.append("?");
+
+ String queryParam = parameters.entrySet().stream()
+ .map(entry -> URLEncoder.encode(entry.getKey()) + "=" + URLEncoder.encode(entry.getValue().toString()))
+ .collect(joining("&"));
+ sb.append(queryParam);
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java
new file mode 100644
index 0000000..01cc0bc
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerConstant.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer;
+
+public class LogviewerConstant {
+ public static final int DEFAULT_BYTES_PER_PAGE = 51200;
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java
new file mode 100644
index 0000000..24ccf6e
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/LogviewerServer.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer;
+
+import com.codahale.metrics.Meter;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.drpc.webapp.ReqContextFilter;
+import org.apache.storm.daemon.wip.logviewer.utils.LogCleaner;
+import org.apache.storm.daemon.wip.logviewer.webapp.LogviewerApplication;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.security.auth.AuthUtils;
+import org.apache.storm.security.auth.IHttpCredentialsPlugin;
+import org.apache.storm.ui.FilterConfiguration;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Utils;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.FilterMapping;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.storm.DaemonConfig.UI_HEADER_BUFFER_BYTES;
+
+public class LogviewerServer implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(LogviewerServer.class);
+ private static final Meter meterShutdownCalls = StormMetricsRegistry.registerMeter("drpc:num-shutdown-calls");
+ public static final String STATIC_RESOURCE_DIRECTORY_PATH = "./public";
+
+ private static Server mkHttpServer(Map<String, Object> conf) {
+ Integer logviewerHttpPort = (Integer) conf.get(DaemonConfig.LOGVIEWER_PORT);
+ Server ret = null;
+ if (logviewerHttpPort != null && logviewerHttpPort >= 0) {
+ LOG.info("Starting Logviewer HTTP servers...");
+ Integer headerBufferSize = ObjectReader.getInt(conf.get(UI_HEADER_BUFFER_BYTES));
+ String filterClass = (String) (conf.get(DaemonConfig.UI_FILTER));
+ @SuppressWarnings("unchecked")
+ Map<String, String> filterParams = (Map<String, String>) (conf.get(DaemonConfig.UI_FILTER_PARAMS));
+ FilterConfiguration filterConfiguration = new FilterConfiguration(filterClass, filterParams);
+ final List<FilterConfiguration> filterConfigurations = Arrays.asList(filterConfiguration);
+
+ final Integer httpsPort = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_HTTPS_PORT), 0);
+ final String httpsKsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PATH));
+ final String httpsKsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PASSWORD));
+ final String httpsKsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_TYPE));
+ final String httpsKeyPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEY_PASSWORD));
+ final String httpsTsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PATH));
+ final String httpsTsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD));
+ final String httpsTsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_TYPE));
+ final Boolean httpsWantClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_WANT_CLIENT_AUTH));
+ final Boolean httpsNeedClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_NEED_CLIENT_AUTH));
+
+ //TODO a better way to do this would be great.
+ LogviewerApplication.setup(conf);
+ ret = UIHelpers.jettyCreateServer(logviewerHttpPort, null, httpsPort);
+
+ UIHelpers.configSsl(ret, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, httpsKeyPassword,
+ httpsTsPath, httpsTsPassword, httpsTsType, httpsNeedClientAuth, httpsWantClientAuth);
+
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+ try {
+ context.setBaseResource(Resource.newResource(STATIC_RESOURCE_DIRECTORY_PATH));
+ } catch (IOException e) {
+ throw new RuntimeException("Can't locate static resource directory " + STATIC_RESOURCE_DIRECTORY_PATH);
+ }
+
+ context.setContextPath("/");
+ ret.setHandler(context);
+
+ ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
+ holderPwd.setInitOrder(1);
+ context.addServlet(holderPwd,"/");
+
+ ServletHolder jerseyServlet = context.addServlet(ServletContainer.class, "/api/v1/*");
+ jerseyServlet.setInitOrder(2);
+ jerseyServlet.setInitParameter("javax.ws.rs.Application", LogviewerApplication.class.getName());
+
+ UIHelpers.configFilters(context, filterConfigurations);
+ }
+ return ret;
+ }
+
+ private final Server httpServer;
+ private boolean closed = false;
+
+ /**
+ * Constructor.
+ * @param conf Logviewer conf for the servers
+ */
+ public LogviewerServer(Map<String, Object> conf) {
+ httpServer = mkHttpServer(conf);
+ }
+
+ @VisibleForTesting
+ void start() throws Exception {
+ LOG.info("Starting Logviewer...");
+ if (httpServer != null) {
+ httpServer.start();
+ }
+ }
+
+ @VisibleForTesting
+ void awaitTermination() throws InterruptedException {
+ httpServer.join();
+ }
+
+ @Override
+ public synchronized void close() {
+ if (!closed) {
+ //This is kind of useless...
+ meterShutdownCalls.mark();
+
+ //TODO this is causing issues...
+ //if (httpServer != null) {
+ // httpServer.destroy();
+ //}
+
+ closed = true;
+ }
+ }
+
+ /**
+ * @return The port the HTTP server is listening on. Not available until {@link #start() } has run.
+ */
+ public int getHttpServerPort() {
+ assert httpServer.getConnectors().length == 1;
+
+ return httpServer.getConnectors()[0].getLocalPort();
+ }
+
+ /**
+ * Main method to start the server.
+ */
+ public static void main(String [] args) throws Exception {
+ Utils.setupDefaultUncaughtExceptionHandler();
+ Map<String, Object> conf = Utils.readStormConfig();
+
+ try (LogviewerServer server = new LogviewerServer(conf);
+ LogCleaner logCleaner = new LogCleaner(conf)) {
+ Utils.addShutdownHookWithForceKillIn1Sec(() -> server.close());
+ logCleaner.start();
+ StormMetricsRegistry.startMetricsReporters(conf);
+ server.start();
+ server.awaitTermination();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java
new file mode 100644
index 0000000..26d1d63
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogDownloadHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.handler;
+
+import org.apache.storm.daemon.wip.logviewer.utils.LogFileDownloader;
+import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
+
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+
+public class LogviewerLogDownloadHandler {
+
+ private final LogFileDownloader logFileDownloadHelper;
+
+ public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, ResourceAuthorizer resourceAuthorizer) {
+ this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, resourceAuthorizer);
+ }
+
+ public Response downloadLogFile(String fileName, String user) throws IOException {
+ return logFileDownloadHelper.downloadFile(fileName, user, false);
+ }
+
+ public Response downloadDaemonLogFile(String fileName, String user) throws IOException {
+ return logFileDownloadHelper.downloadFile(fileName, user, true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a79053/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java
----------------------------------------------------------------------
diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java
new file mode 100644
index 0000000..0e623f5
--- /dev/null
+++ b/storm-webapp/src/main/java/org/apache/storm/daemon/wip/logviewer/handler/LogviewerLogPageHandler.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.wip.logviewer.handler;
+
+import j2html.TagCreator;
+import j2html.tags.DomContent;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.daemon.DirectoryCleaner;
+import org.apache.storm.daemon.utils.StreamUtil;
+import org.apache.storm.daemon.utils.URLBuilder;
+import org.apache.storm.daemon.wip.logviewer.utils.LogviewerResponseBuilder;
+import org.apache.storm.daemon.wip.logviewer.utils.ResourceAuthorizer;
+import org.apache.storm.daemon.wip.logviewer.utils.WorkerLogs;
+import org.apache.storm.ui.InvalidRequestException;
+import org.apache.storm.ui.UIHelpers;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerUtils;
+import org.apache.storm.utils.Utils;
+import org.jooq.lambda.Unchecked;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
+import javax.ws.rs.core.Response;
+
+import static j2html.TagCreator.*;
+import static java.util.stream.Collectors.toCollection;
+import static java.util.stream.Collectors.toList;
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+import static org.apache.storm.daemon.wip.logviewer.LogviewerConstant.DEFAULT_BYTES_PER_PAGE;
+
+public class LogviewerLogPageHandler {
+ private final String logRoot;
+ private final String daemonLogRoot;
+ private final ResourceAuthorizer resourceAuthorizer;
+
+ public LogviewerLogPageHandler(String logRoot, String daemonLogRoot,
+ ResourceAuthorizer resourceAuthorizer) {
+ this.logRoot = logRoot;
+ this.daemonLogRoot = daemonLogRoot;
+ this.resourceAuthorizer = resourceAuthorizer;
+ }
+
+ public Response listLogFiles(String user, Integer port, String topologyId, String callback, String origin) throws IOException {
+ List<File> fileResults = null;
+ if (topologyId == null) {
+ if (port == null) {
+ fileResults = WorkerLogs.getAllLogsForRootDir(new File(logRoot));
+ } else {
+ fileResults = new ArrayList<>();
+
+ File[] logRootFiles = new File(logRoot).listFiles();
+ if (logRootFiles != null) {
+ for (File topoDir : logRootFiles) {
+ File[] topoDirFiles = topoDir.listFiles();
+ if (topoDirFiles != null) {
+ for (File portDir : topoDirFiles) {
+ if (portDir.getName().equals(port.toString())) {
+ fileResults.addAll(DirectoryCleaner.getFilesForDir(portDir));
+ }
+ }
+ }
+ }
+ }
+ }
+ } else {
+ if (port == null) {
+ fileResults = new ArrayList<>();
+
+ File topoDir = new File(logRoot + Utils.FILE_PATH_SEPARATOR + topologyId);
+ if (topoDir.exists()) {
+ File[] topoDirFiles = topoDir.listFiles();
+ if (topoDirFiles != null) {
+ for (File portDir : topoDirFiles) {
+ fileResults.addAll(DirectoryCleaner.getFilesForDir(portDir));
+ }
+ }
+ }
+
+ } else {
+ File portDir = ConfigUtils.getWorkerDirFromRoot(logRoot, topologyId, port);
+ if (portDir.exists()) {
+ fileResults = DirectoryCleaner.getFilesForDir(portDir);
+ }
+ }
+ }
+
+ List<String> files;
+ if (fileResults != null) {
+ files = fileResults.stream()
+ .map(file -> WorkerLogs.getTopologyPortWorkerLog(file))
+ .sorted().collect(toList());
+ } else {
+ files = new ArrayList<>();
+ }
+
+ return LogviewerResponseBuilder.buildSuccessJsonResponse(files, callback, origin);
+ }
+
+ public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ String rootDir = logRoot;
+ if (resourceAuthorizer.isUserAllowedToAccessFile(fileName, user)) {
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ String path = file.getCanonicalPath();
+ boolean isZipFile = path.endsWith(".gz");
+ File topoDir = file.getParentFile().getParentFile();
+
+ if (file.exists() && new File(rootDir).getCanonicalFile().equals(topoDir.getParentFile())) {
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
+
+ SortedSet<File> logFiles;
+ try {
+ logFiles = Arrays.stream(topoDir.listFiles())
+ .flatMap(Unchecked.function(portDir -> DirectoryCleaner.getFilesForDir(portDir).stream()))
+ .filter(File::isFile)
+ .collect(toCollection(TreeSet::new));
+ } catch (UncheckedIOException e) {
+ throw e.getCause();
+ }
+
+ List<String> filesStrWithoutFileParam = logFiles.stream().map(WorkerLogs::getTopologyPortWorkerLog)
+ .filter(fileStr -> !StringUtils.equals(fileName, fileStr)).collect(toList());
+
+ List<String> reorderedFilesStr = new ArrayList<>();
+ reorderedFilesStr.addAll(filesStrWithoutFileParam);
+ reorderedFilesStr.add(fileName);
+
+ length = length != null ? Math.min(10485760, length) : DEFAULT_BYTES_PER_PAGE;
+
+ String logString;
+ if (isTxtFile(fileName)) {
+ logString = escapeHtml(start != null ? pageFile(path, start, length) : pageFile(path, length));
+ } else {
+ logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
+ }
+
+ start = start != null ? start : Long.valueOf(fileLength - length).intValue();
+
+ List<DomContent> bodyContents = new ArrayList<>();
+ if (StringUtils.isNotEmpty(grep)) {
+ String matchedString = String.join("\n", Arrays.stream(logString.split("\n"))
+ .filter(str -> str.contains(grep)).collect(toList()));
+ bodyContents.add(pre(matchedString).withId("logContent"));
+ } else {
+ DomContent pagerData = null;
+ if (isTxtFile(fileName)) {
+ pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "log");
+ }
+
+ bodyContents.add(searchFileForm(fileName, "no"));
+ // list all files for this topology
+ bodyContents.add(logFileSelectionForm(reorderedFilesStr, "log"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ bodyContents.add(downloadLink(fileName));
+ bodyContents.add(pre(logString).withClass("logContent"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ }
+
+ String content = logTemplate(bodyContents, fileName, user).render();
+ return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ } else {
+ if (resourceAuthorizer.getLogUserGroupWhitelist(fileName) != null) {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ } else {
+ return LogviewerResponseBuilder.buildResponseUnautohrizedUser(user);
+ }
+ }
+ }
+
+ public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
+ String rootDir = daemonLogRoot;
+ File file = new File(rootDir, fileName).getCanonicalFile();
+ String path = file.getCanonicalPath();
+ boolean isZipFile = path.endsWith(".gz");
+
+ if (file.exists() && new File(rootDir).getCanonicalFile().equals(file.getParentFile())) {
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(file) : file.length();
+
+ // all types of files included
+ List<File> logFiles = Arrays.stream(new File(rootDir).listFiles())
+ .filter(File::isFile)
+ .collect(toList());
+
+ List<String> filesStrWithoutFileParam = logFiles.stream()
+ .map(File::getName).filter(fName -> !StringUtils.equals(fileName, fName)).collect(toList());
+
+ List<String> reorderedFilesStr = new ArrayList<>();
+ reorderedFilesStr.addAll(filesStrWithoutFileParam);
+ reorderedFilesStr.add(fileName);
+
+ length = length != null ? Math.min(10485760, length) : DEFAULT_BYTES_PER_PAGE;
+
+ String logString;
+ if (isTxtFile(fileName)) {
+ logString = escapeHtml(start != null ? pageFile(path, start, length) : pageFile(path, length));
+ } else {
+ logString = escapeHtml("This is a binary file and cannot display! You may download the full file.");
+ }
+
+ start = start != null ? start : Long.valueOf(fileLength - length).intValue();
+
+ List<DomContent> bodyContents = new ArrayList<>();
+ if (StringUtils.isNotEmpty(grep)) {
+ String matchedString = String.join("\n", Arrays.stream(logString.split("\n"))
+ .filter(str -> str.contains(grep)).collect(toList()));
+ bodyContents.add(pre(matchedString).withId("logContent"));
+ } else {
+ DomContent pagerData = null;
+ if (isTxtFile(fileName)) {
+ pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "daemonlog");
+ }
+
+ bodyContents.add(searchFileForm(fileName, "yes"));
+ // list all daemon logs
+ bodyContents.add(logFileSelectionForm(reorderedFilesStr, "daemonlog"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ bodyContents.add(daemonDownloadLink(fileName));
+ bodyContents.add(pre(logString).withClass("logContent"));
+ if (pagerData != null) {
+ bodyContents.add(pagerData);
+ }
+ }
+
+ String content = logTemplate(bodyContents, fileName, user).render();
+ return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
+ } else {
+ return LogviewerResponseBuilder.buildResponsePageNotFound();
+ }
+ }
+
+ private DomContent logTemplate(List<DomContent> bodyContents, String fileName, String user) {
+ List<DomContent> finalBodyContents = new ArrayList<>();
+
+ if (StringUtils.isNotBlank(user)) {
+ finalBodyContents.add(div(p("User: " + user)).withClass("ui-user"));
+ }
+
+ finalBodyContents.add(div(p("Note: the drop-list shows at most 1024 files for each worker directory.")).withClass("ui-note"));
+ finalBodyContents.add(h3(escapeHtml(fileName)));
+ finalBodyContents.addAll(bodyContents);
+
+ return html(
+ head(
+ title(escapeHtml(fileName) + " - Storm Log Viewer"),
+ link().withRel("stylesheet").withHref("/css/bootstrap-3.3.1.min.css"),
+ link().withRel("stylesheet").withHref("/css/jquery.dataTables.1.10.4.min.css"),
+ link().withRel("stylesheet").withHref("/css/style.css")
+ ),
+ body(
+ finalBodyContents.toArray(new DomContent[]{})
+ )
+ );
+ }
+
+ private DomContent downloadLink(String fileName) {
+ return p(linkTo(UIHelpers.urlFormat("/api/v1/download?file=%s", fileName), "Download Full File"));
+ }
+
+ private DomContent daemonDownloadLink(String fileName) {
+ return p(linkTo(UIHelpers.urlFormat("/api/v1/daemondownload?file=%s", fileName), "Download Full File"));
+ }
+
+ private DomContent linkTo(String url, String content) {
+ return a(content).withHref(url);
+ }
+
+ private DomContent logFileSelectionForm(List<String> logFiles, String type) {
+ return form(
+ dropDown("file", logFiles),
+ input().withType("submit").withValue("Switch file")
+ ).withAction(type).withId("list-of-files");
+ }
+
+ private DomContent dropDown(String name, List<String> logFiles) {
+ List<DomContent> options = logFiles.stream().map(TagCreator::option).collect(toList());
+ return select(options.toArray(new DomContent[]{})).withName(name).withId(name);
+ }
+
+ private DomContent searchFileForm(String fileName, String isDaemonValue) {
+ return form(
+ text("search this file:"),
+ input().withType("text").withName("search"),
+ input().withType("hidden").withName("is-daemon").withValue(isDaemonValue),
+ input().withType("hidden").withName("file").withValue(fileName),
+ input().withType("submit").withValue("Search")
+ ).withAction("/logviewer_search.html").withId("search-box");
+ }
+
+ private DomContent pagerLinks(String fileName, Integer start, Integer length, Integer fileLength, String type) {
+ int prevStart = Math.max(0, start - length);
+ int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length;
+ List<DomContent> btnLinks = new ArrayList<>();
+
+ Map<String, Object> urlQueryParams = new HashMap<>();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("start", Math.max(0, start - length));
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start));
+
+ urlQueryParams.clear();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("start", 0);
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "First"));
+
+ urlQueryParams.clear();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Last"));
+
+ urlQueryParams.clear();
+ urlQueryParams.put("file", fileName);
+ urlQueryParams.put("start", Math.min(Math.max(0, fileLength - length), start + length));
+ urlQueryParams.put("length", length);
+
+ btnLinks.add(toButtonLink(URLBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start));
+
+ return div(btnLinks.toArray(new DomContent[]{}));
+ }
+
+ private DomContent toButtonLink(String url, String text) {
+ return toButtonLink(url, text, true);
+ }
+
+ private DomContent toButtonLink(String url, String text, boolean enabled) {
+ return a(text).withHref(url).withClass("btn btn-default " + (enabled ? "enabled" : "disabled"));
+ }
+
+ private String pageFile(String path, Integer tail) throws IOException, InvalidRequestException {
+ boolean isZipFile = path.endsWith(".gz");
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(new File(path)) : new File(path).length();
+ long skip = fileLength - tail;
+ return pageFile(path, Long.valueOf(skip).intValue(), tail);
+ }
+
+ private String pageFile(String path, Integer start, Integer length) throws IOException, InvalidRequestException {
+ boolean isZipFile = path.endsWith(".gz");
+ long fileLength = isZipFile ? ServerUtils.zipFileSize(new File(path)) : new File(path).length();
+
+ try (InputStream input = isZipFile ? new GZIPInputStream(new FileInputStream(path)) : new FileInputStream(path);
+ ByteArrayOutputStream output = new ByteArrayOutputStream()) {
+ if (start >= fileLength) {
+ throw new InvalidRequestException("Cannot start past the end of the file");
+ }
+ if (start > 0) {
+ StreamUtil.skipBytes(input, start);
+ }
+
+ byte[] buffer = new byte[1024];
+ while (output.size() < length) {
+ int size = input.read(buffer, 0, Math.min(1024, length - output.size()));
+ if (size > 0) {
+ output.write(buffer, 0, size);
+ } else {
+ break;
+ }
+ }
+
+ return output.toString();
+ }
+ }
+
+ private boolean isTxtFile(String fileName) {
+ Pattern p = Pattern.compile("\\.(log.*|txt|yaml|pid)$");
+ Matcher matcher = p.matcher(fileName);
+ return matcher.find();
+ }
+}