You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vi...@apache.org on 2019/03/24 23:34:27 UTC
[drill] branch master updated (5e2251a -> be73250)
This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from 5e2251a DRILL-7110: Skip writing profile when an ALTER SESSION is executed (#1703)
new ef951f5 DRILL-7124: Fix logger for ShowFilesHandler
new 6e98d04 DRILL-7118: Filter not getting pushed down on MapR-DB tables.
new 90b8fcb DRILL-7103: Incorrect conversion to integer in BsonRecordReader#writeTimeStamp
new bd37434 DRILL-7079: Drill can't query views from the S3 storage when plain authentication is enabled
new be73250 DRILL-6562: Plugin Management improvements
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../mapr/db/json/CompareFunctionsProcessor.java | 14 +-
.../apache/drill/exec/dotdrill/DotDrillFile.java | 8 +
.../planner/sql/handlers/ShowFilesHandler.java | 5 +-
.../drill/exec/server/rest/LogsResources.java | 15 +-
.../drill/exec/server/rest/StorageResources.java | 150 +++++++------
.../drill/exec/store/bson/BsonRecordReader.java | 2 +-
exec/java-exec/src/main/resources/rest/options.ftl | 4 +-
.../main/resources/rest/static/js/serverMessage.js | 35 +++
.../src/main/resources/rest/storage/list.ftl | 250 ++++++++++++++++++---
.../src/main/resources/rest/storage/update.ftl | 120 ++++++----
.../exec/store/bson/TestBsonRecordReader.java | 8 +-
11 files changed, 449 insertions(+), 162 deletions(-)
create mode 100644 exec/java-exec/src/main/resources/rest/static/js/serverMessage.js
[drill] 04/05: DRILL-7079: Drill can't query views from the S3
storage when plain authentication is enabled
Posted by vi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit bd374345cf11642857b702ceba660580f3cf4af5
Author: Bohdan Kazydub <bo...@gmail.com>
AuthorDate: Thu Mar 21 10:52:14 2019 +0200
DRILL-7079: Drill can't query views from the S3 storage when plain authentication is enabled
closes #1712
---
.../main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java
index 04b0d74..761d4ac 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.dotdrill;
import org.apache.drill.common.config.LogicalPlanPersistence;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.util.ImpersonationUtil;
import org.apache.hadoop.fs.FileStatus;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
@@ -55,6 +56,13 @@ public class DotDrillFile {
* @return Return owner of the file in underlying file system.
*/
public String getOwner() {
+ if (type == DotDrillType.VIEW && status.getOwner().isEmpty()) {
+ // Drill view S3AFileStatus is not populated with owner (it has default value of "").
+ // This empty String causes IllegalArgumentException to be thrown (if impersonation is enabled) in
+ // SchemaTreeProvider#createRootSchema(String, SchemaConfigInfoProvider). To work-around the issue
+ // we can return current user as if they were the owner of the file (since they have access to it).
+ return ImpersonationUtil.getProcessUserName();
+ }
return status.getOwner();
}
[drill] 01/05: DRILL-7124: Fix logger for ShowFilesHandler
Posted by vi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit ef951f50376f4e1b993cd4788367fc891743ff6b
Author: Arina Ielchiieva <ar...@gmail.com>
AuthorDate: Wed Mar 20 13:03:07 2019 +0200
DRILL-7124: Fix logger for ShowFilesHandler
closes #1705
---
.../org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java
index 3398340..1318e6f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java
@@ -31,14 +31,17 @@ import org.apache.drill.exec.store.ischema.Records;
import org.apache.drill.exec.util.FileSystemUtil;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
import java.sql.Timestamp;
import java.util.List;
import java.util.stream.Collectors;
+import static org.slf4j.LoggerFactory.getLogger;
+
public class ShowFilesHandler extends DefaultSqlHandler {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SetOptionHandler.class);
+ private static final Logger logger = getLogger(ShowFilesHandler.class);
public ShowFilesHandler(SqlHandlerConfig config) {
super(config);
[drill] 05/05: DRILL-6562: Plugin Management improvements
Posted by vi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit be73250e68ffd836541223cf4aca395896362866
Author: Vitalii Diravka <vi...@gmail.com>
AuthorDate: Mon Jul 23 13:03:50 2018 +0300
DRILL-6562: Plugin Management improvements
- allow export plugin configs to json or hocon file formt
- allow export plugins configs for all/enabled/disabled groups
- add modals for export plugins and create new plugin
- storage UI improvements,responsive Storage page
- StorageResources refactoring. Remove redundant deletePlugin() DELETE request
- fix broken message for deletePlugin
closes #1692
---
.../drill/exec/server/rest/LogsResources.java | 15 +-
.../drill/exec/server/rest/StorageResources.java | 150 +++++++------
exec/java-exec/src/main/resources/rest/options.ftl | 4 +-
.../main/resources/rest/static/js/serverMessage.js | 35 +++
.../src/main/resources/rest/storage/list.ftl | 250 ++++++++++++++++++---
.../src/main/resources/rest/storage/update.ftl | 120 ++++++----
6 files changed, 424 insertions(+), 150 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
index 51cf994..4aa2061 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
@@ -35,6 +35,7 @@ import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
@@ -44,7 +45,6 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Collection;
import java.util.LinkedHashMap;
@@ -138,9 +138,9 @@ public class LogsResources {
@Produces(MediaType.TEXT_PLAIN)
public Response getFullLog(@PathParam("name") final String name) {
File file = getFileByName(getLogFolder(), name);
- Response.ResponseBuilder response = Response.ok(file);
- response.header("Content-Disposition", String.format("attachment;filename=\"%s\"", name));
- return response.build();
+ return Response.ok(file)
+ .header(HttpHeaders.CONTENT_DISPOSITION, String.format("attachment;filename=\"%s\"", name))
+ .build();
}
private File getLogFolder() {
@@ -148,12 +148,7 @@ public class LogsResources {
}
private File getFileByName(File folder, final String name) {
- File[] files = folder.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String fileName) {
- return fileName.equals(name);
- }
- });
+ File[] files = folder.listFiles((dir, fileName) -> fileName.equals(name));
if (files.length == 0) {
throw new DrillRuntimeException (name + " doesn't exist");
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
index 1d6d148..8d71bdb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
@@ -19,26 +19,31 @@ package org.apache.drill.exec.server.rest;
import java.io.IOException;
import java.io.StringReader;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import javax.annotation.security.RolesAllowed;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.xml.bind.annotation.XmlRootElement;
+import com.fasterxml.jackson.core.JsonParser;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.server.rest.DrillRestServer.UserAuthEnabled;
@@ -49,7 +54,6 @@ import org.glassfish.jersey.server.mvc.Viewable;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
@@ -63,34 +67,35 @@ public class StorageResources {
@Inject ObjectMapper mapper;
@Inject SecurityContext sc;
- private static final Comparator<PluginConfigWrapper> PLUGIN_COMPARATOR = new Comparator<PluginConfigWrapper>() {
- @Override
- public int compare(PluginConfigWrapper o1, PluginConfigWrapper o2) {
- return o1.getName().compareTo(o2.getName());
- }
- };
+ private static final String JSON_FORMAT = "json";
+ private static final String HOCON_FORMAT = "conf";
+ private static final String ALL_PLUGINS = "all";
+ private static final String ENABLED_PLUGINS = "enabled";
+ private static final String DISABLED_PLUGINS = "disabled";
+
+ private static final Comparator<PluginConfigWrapper> PLUGIN_COMPARATOR =
+ Comparator.comparing(PluginConfigWrapper::getName);
@GET
- @Path("/storage.json")
+ @Path("/storage/{group}/plugins/export/{format}")
@Produces(MediaType.APPLICATION_JSON)
- public List<PluginConfigWrapper> getStoragePluginsJSON() {
-
- List<PluginConfigWrapper> list = Lists.newArrayList();
- for (Map.Entry<String, StoragePluginConfig> entry : Lists.newArrayList(storage.getStore().getAll())) {
- PluginConfigWrapper plugin = new PluginConfigWrapper(entry.getKey(), entry.getValue());
- list.add(plugin);
- }
-
- Collections.sort(list, PLUGIN_COMPARATOR);
-
- return list;
+ public Response getConfigsFor(@PathParam("group") String pluginGroup, @PathParam("format") String format) {
+ return isSupported(format)
+ ? Response.ok()
+ .entity(getConfigsFor(pluginGroup).toArray())
+ .header(HttpHeaders.CONTENT_DISPOSITION, String.format("attachment;filename=\"%s_storage_plugins.%s\"",
+ pluginGroup, format))
+ .build()
+ : Response.status(Response.Status.NOT_FOUND.getStatusCode(),
+ String.format("Unknown file type %s for %s Storage Plugin Configs", format, pluginGroup))
+ .build();
}
@GET
@Path("/storage")
@Produces(MediaType.TEXT_HTML)
- public Viewable getStoragePlugins() {
- List<PluginConfigWrapper> list = getStoragePluginsJSON();
+ public Viewable getPlugins() {
+ List<PluginConfigWrapper> list = getConfigsFor(ALL_PLUGINS);
return ViewableWithPermissions.create(authEnabled.get(), "/rest/storage/list.ftl", sc, list);
}
@@ -98,7 +103,7 @@ public class StorageResources {
@GET
@Path("/storage/{name}.json")
@Produces(MediaType.APPLICATION_JSON)
- public PluginConfigWrapper getStoragePluginJSON(@PathParam("name") String name) {
+ public PluginConfigWrapper getPluginConfig(@PathParam("name") String name) {
try {
// TODO: DRILL-6412: No need to get StoragePlugin. It is enough to have plugin name and config here
StoragePlugin plugin = storage.getPlugin(name);
@@ -106,7 +111,7 @@ public class StorageResources {
return new PluginConfigWrapper(name, plugin.getConfig());
}
} catch (Exception e) {
- logger.info("Failure while trying to access storage config: {}", name, e);
+ logger.error("Failure while trying to access storage config: {}", name, e);
}
return new PluginConfigWrapper(name, null);
}
@@ -114,54 +119,46 @@ public class StorageResources {
@GET
@Path("/storage/{name}")
@Produces(MediaType.TEXT_HTML)
- public Viewable getStoragePlugin(@PathParam("name") String name) {
- PluginConfigWrapper plugin = getStoragePluginJSON(name);
- return ViewableWithPermissions.create(authEnabled.get(), "/rest/storage/update.ftl", sc, plugin);
+ public Viewable getPlugin(@PathParam("name") String name) {
+ return ViewableWithPermissions.create(authEnabled.get(), "/rest/storage/update.ftl", sc,
+ getPluginConfig(name));
}
@GET
@Path("/storage/{name}/enable/{val}")
@Produces(MediaType.APPLICATION_JSON)
public JsonResult enablePlugin(@PathParam("name") String name, @PathParam("val") Boolean enable) {
- PluginConfigWrapper plugin = getStoragePluginJSON(name);
+ PluginConfigWrapper plugin = getPluginConfig(name);
try {
- if (plugin.setEnabledInStorage(storage, enable)) {
- return message("success");
- } else {
- return message("error (plugin does not exist)");
- }
+ return plugin.setEnabledInStorage(storage, enable)
+ ? message("Success")
+ : message("Error (plugin does not exist)");
} catch (ExecutionSetupException e) {
- logger.debug("Error in enabling storage name: " + name + " flag: " + enable);
- return message("error (unable to enable/ disable storage)");
+ logger.debug("Error in enabling storage name: {} flag: {}", name, enable);
+ return message("Error (unable to enable / disable storage)");
}
}
@GET
- @Path("/storage/{name}/export")
- @Produces(MediaType.APPLICATION_JSON)
- public Response exportPlugin(@PathParam("name") String name) {
- Response.ResponseBuilder response = Response.ok(getStoragePluginJSON(name));
- response.header("Content-Disposition", String.format("attachment;filename=\"%s.json\"", name));
- return response.build();
- }
-
- @DELETE
- @Path("/storage/{name}.json")
+ @Path("/storage/{name}/export/{format}")
@Produces(MediaType.APPLICATION_JSON)
- public JsonResult deletePluginJSON(@PathParam("name") String name) {
- PluginConfigWrapper plugin = getStoragePluginJSON(name);
- if (plugin.deleteFromStorage(storage)) {
- return message("success");
- } else {
- return message("error (unable to delete storage)");
- }
+ public Response exportPlugin(@PathParam("name") String name, @PathParam("format") String format) {
+ return isSupported(format)
+ ? Response.ok(getPluginConfig(name))
+ .header(HttpHeaders.CONTENT_DISPOSITION, String.format("attachment;filename=\"%s.%s\"", name, format))
+ .build()
+ : Response.status(Response.Status.NOT_FOUND.getStatusCode(),
+ String.format("Unknown file type %s for Storage Plugin Config: %s", format, name))
+ .build();
}
@GET
@Path("/storage/{name}/delete")
@Produces(MediaType.APPLICATION_JSON)
public JsonResult deletePlugin(@PathParam("name") String name) {
- return deletePluginJSON(name);
+ return getPluginConfig(name).deleteFromStorage(storage)
+ ? message("Success")
+ : message("Error (unable to delete %s storage plugin)", name);
}
@POST
@@ -171,35 +168,62 @@ public class StorageResources {
public JsonResult createOrUpdatePluginJSON(PluginConfigWrapper plugin) {
try {
plugin.createOrUpdateInStorage(storage);
- return message("success");
+ return message("Success");
} catch (ExecutionSetupException e) {
logger.error("Unable to create/ update plugin: " + plugin.getName(), e);
- return message("Error while creating/ updating storage : " + (e.getCause() != null ? e.getCause().getMessage() : e.getMessage()));
+ return message("Error while creating / updating storage : %s", e.getCause() == null ? e.getMessage() :
+ e.getCause().getMessage());
}
}
@POST
- @Path("/storage/{name}")
- @Consumes("application/x-www-form-urlencoded")
+ @Path("/storage/create_update")
+ @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
@Produces(MediaType.APPLICATION_JSON)
public JsonResult createOrUpdatePlugin(@FormParam("name") String name, @FormParam("config") String storagePluginConfig) {
try {
+ mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
StoragePluginConfig config = mapper.readValue(new StringReader(storagePluginConfig), StoragePluginConfig.class);
return createOrUpdatePluginJSON(new PluginConfigWrapper(name, config));
} catch (JsonMappingException e) {
logger.debug("Error in JSON mapping: {}", storagePluginConfig, e);
- return message("error (invalid JSON mapping)");
+ return message("Error (invalid JSON mapping)");
} catch (JsonParseException e) {
logger.debug("Error parsing JSON: {}", storagePluginConfig, e);
- return message("error (unable to parse JSON)");
+ return message("Error (unable to parse JSON)");
} catch (IOException e) {
logger.debug("Failed to read: {}", storagePluginConfig, e);
- return message("error (unable to read)");
+ return message("Error (unable to read)");
}
}
- private JsonResult message(String message) {
- return new JsonResult(message);
+ private JsonResult message(String message, Object... args) {
+ return new JsonResult(String.format(message, args));
+ }
+
+ private boolean isSupported(String format) {
+ return JSON_FORMAT.equalsIgnoreCase(format) || HOCON_FORMAT.equalsIgnoreCase(format);
+ }
+
+ private List<PluginConfigWrapper> getConfigsFor(String pluginGroup) {
+ return StreamSupport.stream(
+ Spliterators.spliteratorUnknownSize(storage.getStore().getAll(), Spliterator.ORDERED), false)
+ .filter(byPluginGroup(pluginGroup))
+ .map(entry -> new PluginConfigWrapper(entry.getKey(), entry.getValue()))
+ .sorted(PLUGIN_COMPARATOR)
+ .collect(Collectors.toList());
+ }
+
+ private Predicate<Map.Entry<String, StoragePluginConfig>> byPluginGroup(String pluginGroup) {
+ if (ALL_PLUGINS.equalsIgnoreCase(pluginGroup)) {
+ return entry -> true;
+ } else if (ENABLED_PLUGINS.equalsIgnoreCase(pluginGroup)) {
+ return entry -> entry.getValue().isEnabled();
+ } else if (DISABLED_PLUGINS.equalsIgnoreCase(pluginGroup)) {
+ return entry -> !entry.getValue().isEnabled();
+ } else {
+ return entry -> false;
+ }
}
@XmlRootElement
diff --git a/exec/java-exec/src/main/resources/rest/options.ftl b/exec/java-exec/src/main/resources/rest/options.ftl
index 9b934e9..d05dd7f 100644
--- a/exec/java-exec/src/main/resources/rest/options.ftl
+++ b/exec/java-exec/src/main/resources/rest/options.ftl
@@ -47,10 +47,10 @@
let optionKind = $("#"+optionName+" input[name='kind']").attr("value");
//Extracting value from the form's INPUT element
let optionValue = $("#"+optionName+" input[name='value']").val();
- if (optionKind == "BOOLEAN") {
+ if (optionKind === "BOOLEAN") {
//Extracting boolean value from the form's SELECT element (since this is a dropdown input)
optionValue = $("#"+optionName+" select[name='value']").val();
- } else if (optionKind != "STRING") { //i.e. it is a number (FLOAT/DOUBLE/LONG)
+ } else if (optionKind !== "STRING") { //i.e. it is a number (FLOAT/DOUBLE/LONG)
if (isNaN(optionValue)) {
let actualOptionName=optionName.replace(/\\\./gi, ".");
let alertValues = {'_numericOption_': optionValue, '_optionName_': actualOptionName };
diff --git a/exec/java-exec/src/main/resources/rest/static/js/serverMessage.js b/exec/java-exec/src/main/resources/rest/static/js/serverMessage.js
new file mode 100644
index 0000000..aeb9960
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/static/js/serverMessage.js
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership. The ASF licenses this file to
+ * You under the Apache License, Version 2.0 (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of
+ * the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ * by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ * OF ANY KIND, either express or implied. See the License for the specific
+ * language governing permissions and limitations under the License.
+ */
+
+// Shows Json message from the server
+function serverMessage(data) {
+ const messageEl = $("#message");
+ if (data.result === "Success") {
+ messageEl.removeClass("hidden")
+ .removeClass("alert-danger")
+ .addClass("alert-info")
+ .text(data.result).alert();
+ setTimeout(function() { location.reload(); }, 800);
+ } else {
+ messageEl.addClass("hidden");
+ // Wait a fraction of a second before showing the message again. This
+ // makes it clear if a second attempt gives the same error as
+ // the first that a "new" message came back from the server
+ setTimeout(function() {
+ messageEl.removeClass("hidden")
+ .removeClass("alert-info")
+ .addClass("alert-danger")
+ .text("Please retry: " + data.result).alert();
+ }, 200);
+ }
+}
diff --git a/exec/java-exec/src/main/resources/rest/storage/list.ftl b/exec/java-exec/src/main/resources/rest/storage/list.ftl
index 016c019..c821a9d 100644
--- a/exec/java-exec/src/main/resources/rest/storage/list.ftl
+++ b/exec/java-exec/src/main/resources/rest/storage/list.ftl
@@ -17,27 +17,60 @@
limitations under the License.
-->
+
<#include "*/generic.ftl">
<#macro page_head>
+ <script src="/static/js/jquery.form.js"></script>
+
+ <!-- Ace Libraries for Syntax Formatting -->
+ <script src="/static/js/ace-code-editor/ace.js" type="text/javascript" charset="utf-8"></script>
+ <script src="/static/js/ace-code-editor/theme-eclipse.js" type="text/javascript" charset="utf-8"></script>
+ <script src="/static/js/serverMessage.js"></script>
</#macro>
<#macro page_body>
<div class="page-header">
</div>
- <h4>Enabled Storage Plugins</h4>
- <div class="table-responsive">
- <table class="table">
+
+ <h4 class="col-xs-6">Plugin Management</h4>
+ <table style="margin: 10px" class="table">
+ <tbody>
+ <tr>
+ <td style="border:none;">
+ <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#new-plugin-modal">
+ Create
+ </button>
+ <button type="button" class="btn btn-primary" name="all" data-toggle="modal" data-target="#pluginsModal">
+ Export all
+ </button>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
+ <div class="page-header" style="margin: 5px;"></div>
+
+ <div class="table-responsive col-sm-12 col-md-6 col-lg-5 col-xl-5">
+ <h4>Enabled Storage Plugins</h4>
+ <table class="table table-hover">
<tbody>
<#list model as plugin>
<#if plugin.enabled() == true>
<tr>
- <td style="border:none; width:200px;">
+ <td style="border:none; max-width: 200px; overflow: hidden; text-overflow: ellipsis;">
${plugin.getName()}
</td>
<td style="border:none;">
- <a class="btn btn-primary" href="/storage/${plugin.getName()}">Update</a>
- <a class="btn btn-default" onclick="doEnable('${plugin.getName()}', false)">Disable</a>
- <a class="btn btn-default" href="/storage/${plugin.getName()}/export"">Export</a>
+ <button type="button" class="btn btn-primary" onclick="location.href='/storage/${plugin.getName()}'">
+ Update
+ </button>
+ <button type="button" class="btn btn-warning" onclick="doEnable('${plugin.getName()}', false)">
+ Disable
+ </button>
+ <button type="button" class="btn" name="${plugin.getName()}" data-toggle="modal"
+ data-target="#pluginsModal">
+ Export
+ </button>
</td>
</tr>
</#if>
@@ -45,21 +78,28 @@
</tbody>
</table>
</div>
- <div class="page-header">
- </div>
- <h4>Disabled Storage Plugins</h4>
- <div class="table-responsive">
- <table class="table">
+
+ <div class="table-responsive col-sm-12 col-md-6 col-lg-7 col-xl-7">
+ <h4>Disabled Storage Plugins</h4>
+ <table class="table table-hover">
<tbody>
<#list model as plugin>
<#if plugin.enabled() == false>
<tr>
- <td style="border:none; width:200px;">
+ <td style="border:none; max-width: 200px; overflow: hidden; text-overflow: ellipsis;">
${plugin.getName()}
</td>
<td style="border:none;">
- <a class="btn btn-primary" href="/storage/${plugin.getName()}">Update</a>
- <a class="btn btn-primary" onclick="doEnable('${plugin.getName()}', true)">Enable</a>
+ <button type="button" class="btn btn-primary" onclick="location.href='/storage/${plugin.getName()}'">
+ Update
+ </button>
+ <button type="button" class="btn btn-success" onclick="doEnable('${plugin.getName()}', true)">
+ Enable
+ </button>
+ <button type="button" class="btn" name="${plugin.getName()}" data-toggle="modal"
+ data-target="#pluginsModal">
+ Export
+ </button>
</td>
</tr>
</#if>
@@ -67,29 +107,175 @@
</tbody>
</table>
</div>
- <div class="page-header">
+
+
+ <#-- Modal window for exporting plugin config (including group plugins modal) -->
+ <div class="modal fade" id="pluginsModal" tabindex="-1" role="dialog" aria-labelledby="exportPlugin" aria-hidden="true">
+ <div class="modal-dialog modal-sm" role="document">
+ <div class="modal-content">
+ <div class="modal-header">
+ <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
+ <h4 class="modal-title" id="exportPlugin">Plugin config</h4>
+ </div>
+ <div class="modal-body">
+ <div id="format" style="display: inline-block; position: relative;">
+ <label for="format">Format</label>
+ <div class="radio">
+ <label>
+ <input type="radio" name="format" id="json" value="json" checked="checked">
+ JSON
+ </label>
+ </div>
+ <div class="radio">
+ <label>
+ <input type="radio" name="format" id="hocon" value="conf">
+ HOCON
+ </label>
+ </div>
+ </div>
+
+ <div id="plugin-set" class="" style="display: inline-block; position: relative; float: right;">
+ <label for="format">Plugin group</label>
+ <div class="radio">
+ <label>
+ <input type="radio" name="group" id="all" value="all" checked="checked">
+ ALL
+ </label>
+ </div>
+ <div class="radio">
+ <label>
+ <input type="radio" name="group" id="enabled" value="enabled">
+ ENABLED
+ </label>
+ </div>
+ <div class="radio">
+ <label>
+ <input type="radio" name="group" id="disabled" value="disabled">
+ DISABLED
+ </label>
+ </div>
+ </div>
+ </div>
+
+ <div class="modal-footer">
+ <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+ <button type="button" id="export" class="btn btn-primary">Export</button>
+ </div>
+ </div>
+ </div>
</div>
- <div>
- <h4>New Storage Plugin</h4>
- <form class="form-inline" id="newStorage" role="form" action="/" method="GET">
- <div class="form-group">
- <input type="text" class="form-control" id="storageName" placeholder="Storage Name">
+ <#-- Modal window for exporting plugin config (including group plugins modal) -->
+
+ <#-- Modal window for creating plugin -->
+ <div class="modal fade" id="new-plugin-modal" role="dialog" aria-labelledby="configuration">
+ <div class="modal-dialog" role="document">
+ <div class="modal-content">
+ <div class="modal-header">
+ <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
+ <h4 class="modal-title" id="configuration">New Storage Plugin</h4>
+ </div>
+ <div class="modal-body">
+
+ <form id="createForm" role="form" action="/storage/create_update" method="POST">
+ <input type="text" class="form-control" name="name" placeholder="Storage Name">
+ <h3>Configuration</h3>
+ <div class="form-group">
+ <div id="editor" class="form-control"></div>
+ <textarea class="form-control" id="config" name="config" data-editor="json" style="display: none;">
+ </textarea>
+ </div>
+ <div style="text-align: right; margin: 10px">
+ <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+ <button type="submit" class="btn btn-primary" onclick="doCreate()">Create</button>
+ </div>
+ </form>
+
+ <div id="message" class="hidden alert alert-info">
+ </div>
+ </div>
</div>
- <button type="submit" class="btn btn-default" onclick="doSubmit()">Create</button>
- </form>
+ </div>
</div>
+ <#-- Modal window for creating plugin -->
+
<script>
- function doSubmit() {
- var name = document.getElementById("storageName");
- var form = document.getElementById("newStorage");
- form.action = "/storage/" + name.value;
- form.submit();
- };
function doEnable(name, flag) {
- $.get("/storage/" + name + "/enable/" + flag, function(data) {
- location.reload();
+ if (flag || confirm(name + ' plugin will be disabled')) {
+ $.get("/storage/" + name + "/enable/" + flag, function() {
+ location.reload();
+ });
+ }
+ }
+
+ function doCreate() {
+ $("#createForm").ajaxForm({
+ dataType: 'json',
+ success: serverMessage
+ });
+ }
+
+ // Formatting create plugin textarea
+ $('#new-plugin-modal').on('show.bs.modal', function() {
+ const editor = ace.edit("editor");
+ const textarea = $('textarea[name="config"]');
+
+ editor.setAutoScrollEditorIntoView(true);
+ editor.setOption("maxLines", 25);
+ editor.setOption("minLines", 10);
+ editor.renderer.setShowGutter(true);
+ editor.renderer.setOption('showLineNumbers', true);
+ editor.renderer.setOption('showPrintMargin', false);
+ editor.getSession().setMode("ace/mode/json");
+ editor.setTheme("ace/theme/eclipse");
+
+ // copy back to textarea on form submit...
+ editor.getSession().on('change', function(){
+ textarea.val(editor.getSession().getValue());
+ });
+ });
+
+ // Modal windows management
+ let exportInstance; // global variable
+ $('#pluginsModal').on('show.bs.modal', function(event) {
+ console.log("alarm");
+ const button = $(event.relatedTarget); // Button that triggered the modal
+ const modal = $(this);
+ exportInstance = button.attr("name");
+
+ const optionalBlock = modal.find('#plugin-set');
+ if (exportInstance === "all") {
+ optionalBlock.removeClass('hide');
+ modal.find('.modal-title').text('Export all Plugins configs');
+ } else {
+ modal.find('#plugin-set').addClass('hide');
+ modal.find('.modal-title').text(exportInstance.toUpperCase() + ' Plugin config');
+ }
+
+ modal.find('#export').click(function() {
+ let format;
+ if (modal.find('#json').is(":checked")) {
+ format = 'json';
+ }
+ if (modal.find('#hocon').is(":checked")) {
+ format = 'conf';
+ }
+ let url;
+ if (exportInstance === "all") {
+ let pluginGroup = "";
+ if (modal.find('#all').is(":checked")) {
+ pluginGroup = 'all';
+ } else if (modal.find('#enabled').is(":checked")) {
+ pluginGroup = 'enabled';
+ } else if (modal.find('#disabled').is(":checked")) {
+ pluginGroup = 'disabled';
+ }
+ url = '/storage/' + pluginGroup + '/plugins/export/' + format;
+ } else {
+ url = '/storage/' + exportInstance + '/export/' + format;
+ }
+ window.open(url);
});
- };
+ });
</script>
</#macro>
diff --git a/exec/java-exec/src/main/resources/rest/storage/update.ftl b/exec/java-exec/src/main/resources/rest/storage/update.ftl
index 4247963..3e827f7 100644
--- a/exec/java-exec/src/main/resources/rest/storage/update.ftl
+++ b/exec/java-exec/src/main/resources/rest/storage/update.ftl
@@ -17,20 +17,21 @@
limitations under the License.
-->
+
<#include "*/generic.ftl">
<#macro page_head>
<script src="/static/js/jquery.form.js"></script>
-
<!-- Ace Libraries for Syntax Formatting -->
<script src="/static/js/ace-code-editor/ace.js" type="text/javascript" charset="utf-8"></script>
<script src="/static/js/ace-code-editor/theme-eclipse.js" type="text/javascript" charset="utf-8"></script>
+ <script src="/static/js/serverMessage.js"></script>
</#macro>
<#macro page_body>
<div class="page-header">
</div>
<h3>Configuration</h3>
- <form id="updateForm" role="form" action="/storage/${model.getName()}" method="POST">
+ <form id="updateForm" role="form" action="/storage/create_update" method="POST">
<input type="hidden" name="name" value="${model.getName()}" />
<div class="form-group">
<div id="editor" class="form-control"></div>
@@ -38,26 +39,58 @@
</textarea>
</div>
<a class="btn btn-default" href="/storage">Back</a>
- <button class="btn btn-default" type="submit" onclick="doUpdate();">
- <#if model.exists()>Update<#else>Create</#if>
- </button>
- <#if model.exists()>
- <#if model.enabled()>
- <a id="enabled" class="btn btn-default">Disable</a>
- <#else>
- <a id="enabled" class="btn btn-primary">Enable</a>
- </#if>
- <a class="btn btn-default" href="/storage/${model.getName()}/export"">Export</a>
- <a id="del" class="btn btn-danger" onclick="deleteFunction()">Delete</a>
+ <button class="btn btn-default" type="submit" onclick="doUpdate();">Update</button>
+ <#if model.enabled()>
+ <a id="enabled" class="btn btn-default">Disable</a>
+ <#else>
+ <a id="enabled" class="btn btn-primary">Enable</a>
</#if>
+ <button type="button" class="btn btn-default export" name="${model.getName()}" data-toggle="modal"
+ data-target="#pluginsModal">
+ Export
+ </button>
+ <a id="del" class="btn btn-danger" onclick="deleteFunction()">Delete</a>
</form>
<br>
<div id="message" class="hidden alert alert-info">
</div>
- <script>
- var editor = ace.edit("editor");
- var textarea = $('textarea[name="config"]');
+ <#-- Modal window-->
+ <div class="modal fade" id="pluginsModal" tabindex="-1" role="dialog" aria-labelledby="exportPlugin" aria-hidden="true">
+ <div class="modal-dialog modal-sm" role="document">
+ <div class="modal-content">
+ <div class="modal-header">
+ <button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
+ <h4 class="modal-title" id="exportPlugin">Plugin config</h4>
+ </div>
+ <div class="modal-body">
+ <div id="format" style="display: inline-block; position: relative;">
+ <label for="format">Format</label>
+ <div class="radio">
+ <label>
+ <input type="radio" name="format" id="json" value="json" checked="checked">
+ JSON
+ </label>
+ </div>
+ <div class="radio">
+ <label>
+ <input type="radio" name="format" id="hocon" value="conf">
+ HOCON
+ </label>
+ </div>
+ </div>
+ </div>
+ <div class="modal-footer">
+ <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+ <button type="button" id="export" class="btn btn-primary">Export</button>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <script>
+ const editor = ace.edit("editor");
+ const textarea = $('textarea[name="config"]');
editor.setAutoScrollEditorIntoView(true);
editor.setOption("maxLines", 25);
@@ -86,36 +119,37 @@
});
});
function doUpdate() {
- $("#updateForm").ajaxForm(function(data) {
- var messageEl = $("#message");
- if (data.result == "success") {
- messageEl.removeClass("hidden")
- .removeClass("alert-danger")
- .addClass("alert-info")
- .text(data.result).alert();
- setTimeout(function() { location.reload(); }, 800);
- } else {
- messageEl.addClass("hidden");
- // Wait a fraction of a second before showing the message again. This
- // makes it clear if a second attempt gives the same error as
- // the first that a "new" message came back from the server
- setTimeout(function() {
- messageEl.removeClass("hidden")
- .removeClass("alert-info")
- .addClass("alert-danger")
- .text("Please retry: " + data.result).alert();
- }, 200);
- }
+ $("#updateForm").ajaxForm({
+ dataType: 'json',
+ success: serverMessage
});
- };
+ }
+
function deleteFunction() {
- var temp = confirm("Are you sure?");
- if (temp == true) {
- $.get("/storage/${model.getName()}/delete", function(data) {
- window.location.href = "/storage";
- });
+ if (confirm("Are you sure?")) {
+ $.get("/storage/${model.getName()}/delete", serverMessage);
}
- };
+ }
+
+ // Modal window management
+ $('#pluginsModal').on('show.bs.modal', function (event) {
+ const button = $(event.relatedTarget); // Button that triggered the modal
+ let exportInstance = button.attr("name");
+ const modal = $(this);
+ modal.find('.modal-title').text(exportInstance.toUpperCase() +' Plugin configs');
+ modal.find('.btn-primary').click(function(){
+ let format = "";
+ if (modal.find('#json').is(":checked")) {
+ format = 'json';
+ }
+ if (modal.find('#hocon').is(":checked")) {
+ format = 'conf';
+ }
+
+ let url = '/storage/' + exportInstance + '/export/' + format;
+ window.open(url);
+ });
+ })
</script>
</#macro>
[drill] 03/05: DRILL-7103: Incorrect conversion to integer in
BsonRecordReader#writeTimeStamp
Posted by vi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 90b8fcb8f01277de10d1cdb8c3ecca3ad8a6294d
Author: Oleg Zinoviev <oz...@solit-clouds.ru>
AuthorDate: Thu Mar 14 16:56:45 2019 +0300
DRILL-7103: Incorrect conversion to integer in BsonRecordReader#writeTimeStamp
closes #1695
---
.../java/org/apache/drill/exec/store/bson/BsonRecordReader.java | 2 +-
.../org/apache/drill/exec/store/bson/TestBsonRecordReader.java | 8 +++++---
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/bson/BsonRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/bson/BsonRecordReader.java
index 2580010..64c6200 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/bson/BsonRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/bson/BsonRecordReader.java
@@ -255,7 +255,7 @@ public class BsonRecordReader {
} else {
t = writer.list.timeStamp();
}
- t.writeTimeStamp((int) (dateTime.withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis()));
+ t.writeTimeStamp(dateTime.withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis());
}
private void writeString(String readString, final MapOrListWriterImpl writer, String fieldName, boolean isList) {
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java
index 256f1ee..c3ff89a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/bson/TestBsonRecordReader.java
@@ -75,17 +75,19 @@ public class TestBsonRecordReader {
writer.reset();
bsonReader.write(writer, new BsonDocumentReader(bsonDoc));
SingleMapReaderImpl mapReader = (SingleMapReaderImpl) writer.getMapVector().getReader();
- assertEquals(10l, mapReader.reader("seqNo").readLong().longValue());
+ assertEquals(10L, mapReader.reader("seqNo").readLong().longValue());
}
@Test
public void testTimeStampType() throws IOException {
BsonDocument bsonDoc = new BsonDocument();
- bsonDoc.append("ts", new BsonTimestamp(1000, 10));
+ bsonDoc.append("ts_small", new BsonTimestamp(1000, 10));
+ bsonDoc.append("ts_large", new BsonTimestamp(1000000000, 10));
writer.reset();
bsonReader.write(writer, new BsonDocumentReader(bsonDoc));
SingleMapReaderImpl mapReader = (SingleMapReaderImpl) writer.getMapVector().getReader();
- assertEquals(1000000l, mapReader.reader("ts").readLocalDateTime().atZone(ZoneOffset.systemDefault()).toInstant().toEpochMilli());
+ assertEquals(1000000L, mapReader.reader("ts_small").readLocalDateTime().atZone(ZoneOffset.systemDefault()).toInstant().toEpochMilli());
+ assertEquals(1000000000000L, mapReader.reader("ts_large").readLocalDateTime().atZone(ZoneOffset.systemDefault()).toInstant().toEpochMilli());
}
@Test
[drill] 02/05: DRILL-7118: Filter not getting pushed down on
MapR-DB tables.
Posted by vi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vitalii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 6e98d04d06fa6c3574f857cc66a5b60135c36d40
Author: HanumathRao <ha...@gmail.com>
AuthorDate: Wed Mar 20 12:01:11 2019 -0700
DRILL-7118: Filter not getting pushed down on MapR-DB tables.
closes #1708
---
.../exec/store/mapr/db/json/CompareFunctionsProcessor.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java
index d9db7bc..3ece894 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java
@@ -107,13 +107,13 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
LogicalExpression nameArg = call.args.get(0);
LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null;
- if (valueArg != null) {
- if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
- LogicalExpression swapArg = valueArg;
- valueArg = nameArg;
- nameArg = swapArg;
- evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName);
- }
+ if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
+ LogicalExpression swapArg = valueArg;
+ valueArg = nameArg;
+ nameArg = swapArg;
+ evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName);
+ }
+ if (nameArg != null) {
evaluator.success = nameArg.accept(evaluator, valueArg);
}