You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2016/05/03 19:50:48 UTC
[5/5] drill git commit: DRILL-4571: Add link to local Drill logs from
the web UI
DRILL-4571: Add link to local Drill logs from the web UI
This closes #472
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1a89a7fe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1a89a7fe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1a89a7fe
Branch: refs/heads/master
Commit: 1a89a7fe56533ecea8f7a7c9be6a3699925f3c96
Parents: 94b8aec
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Thu Mar 31 18:43:25 2016 +0300
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue May 3 10:50:09 2016 -0700
----------------------------------------------------------------------
distribution/src/resources/logback.xml | 2 +-
.../org/apache/drill/exec/ExecConstants.java | 5 +
.../server/options/SystemOptionManager.java | 3 +-
.../drill/exec/server/rest/DrillRestServer.java | 1 +
.../drill/exec/server/rest/LogsResources.java | 211 +++++++++++++++++++
.../server/rest/ViewableWithPermissions.java | 1 +
.../src/main/resources/rest/generic.ftl | 3 +
.../src/main/resources/rest/logs/list.ftl | 55 +++++
.../src/main/resources/rest/logs/log.ftl | 37 ++++
9 files changed, 316 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/distribution/src/resources/logback.xml
----------------------------------------------------------------------
diff --git a/distribution/src/resources/logback.xml b/distribution/src/resources/logback.xml
index 350383a..fb53dfc 100644
--- a/distribution/src/resources/logback.xml
+++ b/distribution/src/resources/logback.xml
@@ -44,7 +44,7 @@
<maxFileSize>100MB</maxFileSize>
</triggeringPolicy>
<encoder>
- <pattern>%msg</pattern>
+ <pattern>%msg%n</pattern>
</encoder>
</appender>
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 7f216f0..17fbb7b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -303,4 +303,9 @@ public interface ExecConstants {
StringValidator IMPERSONATION_POLICY_VALIDATOR =
new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY, "[]");
+ /**
+ * Web settings
+ */
+ String WEB_LOGS_MAX_LINES = "web.logs.max_lines";
+ OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE, 10000);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index db78108..c35ed0e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -135,7 +135,8 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR,
ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR,
ExecConstants.ENABLE_NEW_TEXT_READER,
- ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST
+ ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST,
+ ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR
};
final Map<String, OptionValidator> tmp = new HashMap<>();
for (final OptionValidator validator : validators) {
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
index ceecdb4..0401d58 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
@@ -55,6 +55,7 @@ public class DrillRestServer extends ResourceConfig {
register(QueryResources.class);
register(MetricsResources.class);
register(ThreadsResources.class);
+ register(LogsResources.class);
register(FreemarkerMvcFeature.class);
register(MultiPartFeature.class);
property(ServerProperties.METAINF_SERVICES_LOOKUP_DISABLE, true);
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
new file mode 100644
index 0000000..8a89d41
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
@@ -0,0 +1,211 @@
+/**
+ * ****************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ****************************************************************************
+ */
+package org.apache.drill.exec.server.rest;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Sets;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.work.WorkManager;
+import org.glassfish.jersey.server.mvc.Viewable;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import javax.annotation.security.RolesAllowed;
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
+
+@Path("/")
+@RolesAllowed(ADMIN_ROLE)
+public class LogsResources {
+
+ @Inject DrillRestServer.UserAuthEnabled authEnabled;
+ @Inject SecurityContext sc;
+ @Inject WorkManager work;
+
+ private static final FileFilter file_filter = new FileFilter() {
+ @Override
+ public boolean accept(File file) {
+ return file.isFile();
+ }
+ };
+ private static final DateTimeFormatter format = DateTimeFormat.forPattern("MM/dd/yyyy HH:mm:ss");
+
+
+ @GET
+ @Path("/logs")
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getLogs() {
+ Set<Log> logs = getLogsJSON();
+ return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/list.ftl", sc, logs);
+ }
+
+ @GET
+ @Path("/logs.json")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<Log> getLogsJSON() {
+ Set<Log> logs = Sets.newTreeSet();
+ File[] files = getLogFolder().listFiles(file_filter);
+
+ for (File file : files) {
+ logs.add(new Log(file.getName(), file.length(), file.lastModified()));
+ }
+
+ return logs;
+ }
+
+ @GET
+ @Path("/log/{name}/content")
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getLog(@PathParam("name") String name) throws IOException {
+ LogContent content = getLogJSON(name);
+ return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/log.ftl", sc, content);
+ }
+
+ @GET
+ @Path("/log/{name}/content.json")
+ @Produces(MediaType.APPLICATION_JSON)
+ public LogContent getLogJSON(@PathParam("name") final String name) throws IOException {
+ File file = getFileByName(getLogFolder(), name);
+
+ final int maxLines = work.getContext().getOptionManager().getOption(ExecConstants.WEB_LOGS_MAX_LINES).num_val.intValue();
+
+ try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+ Map<String, String> cache = new LinkedHashMap<String, String>(maxLines, .75f, true) {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
+ return size() > maxLines;
+ }
+ };
+
+ String line;
+ while ((line = br.readLine()) != null) {
+ cache.put(line, null);
+ }
+
+ return new LogContent(file.getName(), cache.keySet(), maxLines);
+ }
+ }
+
+ @GET
+ @Path("/log/{name}/download")
+ @Produces(MediaType.TEXT_PLAIN)
+ public Response getFullLog(@PathParam("name") final String name) {
+ File file = getFileByName(getLogFolder(), name);
+ Response.ResponseBuilder response = Response.ok(file);
+ response.header("Content-Disposition", String.format("attachment;filename\"%s\"", name));
+ return response.build();
+ }
+
+ private File getLogFolder() {
+ return new File(System.getenv("DRILL_LOG_DIR"));
+ }
+
+ private File getFileByName(File folder, final String name) {
+ File[] files = folder.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String fileName) {
+ return fileName.equals(name);
+ }
+ });
+ if (files.length == 0) {
+ throw new DrillRuntimeException (name + " doesn't exist");
+ }
+ return files[0];
+ }
+
+
+ @XmlRootElement
+ public class Log implements Comparable<Log> {
+
+ private String name;
+ private long size;
+ private DateTime lastModified;
+
+ @JsonCreator
+ public Log (@JsonProperty("name") String name, @JsonProperty("size") long size, @JsonProperty("lastModified") long lastModified) {
+ this.name = name;
+ this.size = size;
+ this.lastModified = new DateTime(lastModified);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getSize() {
+ return Math.ceil(size / 1024d) + " KB";
+ }
+
+ public String getLastModified() {
+ return lastModified.toString(format);
+ }
+
+ @Override
+ public int compareTo(Log log) {
+ return this.getName().compareTo(log.getName());
+ }
+ }
+
+ @XmlRootElement
+ public class LogContent {
+ private String name;
+ private Collection<String> lines;
+ private int maxLines;
+
+ @JsonCreator
+ public LogContent (@JsonProperty("name") String name, @JsonProperty("lines") Collection<String> lines, @JsonProperty("maxLines") int maxLines) {
+ this.name = name;
+ this.lines = lines;
+ this.maxLines = maxLines;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Collection<String> getLines() { return lines; }
+
+ public int getMaxLines() { return maxLines; }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
index b2a0fae..73019aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
@@ -78,6 +78,7 @@ public class ViewableWithPermissions extends Viewable {
.put("showStorage", isAdmin)
.put("showOptions", isAdmin)
.put("showThreads", isAdmin)
+ .put("showLogs", isAdmin)
.put("showLogin", authEnabled && showControls && !isUserLoggedIn)
.put("showLogout", authEnabled && showControls && isUserLoggedIn)
.put("loggedInUserName", authEnabled && showControls &&
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/generic.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl
index b3e249e..60869e7 100644
--- a/exec/java-exec/src/main/resources/rest/generic.ftl
+++ b/exec/java-exec/src/main/resources/rest/generic.ftl
@@ -64,6 +64,9 @@
<#if showThreads == true>
<li><a href="/threads">Threads</a></li>
</#if>
+ <#if showLogs == true>
+ <li><a href="/logs">Logs</a></li>
+ </#if>
</ul>
</#if>
<ul class="nav navbar-nav navbar-right">
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/list.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/logs/list.ftl b/exec/java-exec/src/main/resources/rest/logs/list.ftl
new file mode 100644
index 0000000..3d836df
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/logs/list.ftl
@@ -0,0 +1,55 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+<a href="/queries">back</a><br/>
+<div class="page-header">
+</div>
+
+<#if (model?size > 0)>
+<div class="table-responsive">
+ <table class="table table-hover">
+ <thead>
+ <td>Name</td>
+ <td>Size</td>
+ <td>Last Modified</td>
+ </thead>
+ <tbody>
+ <#list model as log>
+ <tr>
+ <td>
+ <a href="/log/${log.getName()}/content">
+ <div style="height:100%;width:100%;white-space:pre-line">${log.getName()}</div>
+ </a>
+ </td>
+ <td>
+ <div style="height:100%;width:100%;white-space:pre-line">${log.getSize()}</div>
+ </td>
+ <td>
+ <div style="height:100%;width:100%;white-space:pre-line">${log.getLastModified()}</div>
+ </td>
+ </tr>
+ </#list>
+ </tbody>
+ </table>
+</div>
+<#else>
+<div id="message" class="alert alert-info">
+ <strong>No logs are available.</strong>
+</div>
+</#if>
+</#macro>
+
+<@page_html/>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/log.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/logs/log.ftl b/exec/java-exec/src/main/resources/rest/logs/log.ftl
new file mode 100644
index 0000000..b09b57a
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/logs/log.ftl
@@ -0,0 +1,37 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+<a href="/logs">back</a><br/>
+<div class="page-header">
+</div>
+<h3>${model.getName()} <span class="badge alert-info">(last ${model.getMaxLines()} lines)</span></h3>
+<p>
+ <a href="/log/${model.getName()}/download">Download Full Log</a>
+</p>
+ <#if (model.getLines()?size > 0)>
+ <pre>
+ <#list model.getLines() as line>
+${line}
+ </#list>
+ </pre>
+ <#else>
+ <div id="message" class="alert alert-info">
+ <strong>Log is empty.</strong>
+ </div>
+ </#if>
+</#macro>
+
+<@page_html/>
\ No newline at end of file